First commit - moving from LuaForge to SourceForge

350 files changed, 69747 insertions, 0 deletions

diff --git a/src/fftw3/api/api.h b/src/fftw3/api/api.h
new file mode 100644
index 0000000..6b37151
--- /dev/null
+++ b/src/fftw3/api/api.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* internal API definitions */
+#ifndef __API_H__
+#define __API_H__
+
+/* just in case: force <fftw3.h> not to use C99 complex numbers */
+#undef _Complex_I
+
+#include "fftw3.h"
+#include "ifftw.h"
+
+/* the API ``plan'' contains both the kernel plan and problem */
+struct X(plan_s) {
+     plan *pln;
+     problem *prb;
+     int sign;
+};
+
+/* shorthand */
+typedef struct X(plan_s) apiplan;
+
+/* complex type for internal use */
+typedef R C[2];
+
+void X(extract_reim)(int sign, C *c, R **r, R **i);
+
+#define TAINT_UNALIGNED(p, flg) TAINT(p, ((flg) & FFTW_UNALIGNED) != 0)
+
+tensor *X(mktensor_rowmajor)(int rnk, const int *n,
+			     const int *niphys, const int *nophys,
+			     int is, int os);
+
+tensor *X(mktensor_iodims)(int rank, const X(iodim) *dims, int is, int os);
+const int *X(rdft2_pad)(int rnk, const int *n, const int *nembed,
+			int inplace, int cmplx, int **nfree);
+
+int X(many_kosherp)(int rnk, const int *n, int howmany);
+int X(guru_kosherp)(int rank, const X(iodim) *dims,
+		    int howmany_rank, const X(iodim) *howmany_dims);
+
+
+printer *X(mkprinter_file)(FILE *f);
+
+planner *X(the_planner)(void);
+void X(configure_planner)(planner *plnr);
+
+void X(mapflags)(planner *, unsigned);
+
+apiplan *X(mkapiplan)(int sign, unsigned flags, problem *prb);
+
+#endif				/* __API_H__ */
diff --git a/src/fftw3/api/apiplan.c b/src/fftw3/api/apiplan.c
new file mode 100644
index 0000000..b9fee6b
--- /dev/null
+++ b/src/fftw3/api/apiplan.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+
+apiplan *X(mkapiplan)(int sign, unsigned flags, problem *prb)
+WITH_ALIGNED_STACK({
+     plan *pln;
+     plan *pln0;
+     apiplan *p = 0;
+     planner *plnr = X(the_planner)();
+     
+     /* map API flags into FFTW flags */
+     X(mapflags)(plnr, flags);
+     
+     /* create plan */
+     plnr->planner_flags &= ~BLESSING;
+     pln = plnr->adt->mkplan(plnr, prb);
+     
+     if (pln) {
+	  AWAKE(pln, 1);
+	  
+	  /* build apiplan */
+	  p = (apiplan *) MALLOC(sizeof(apiplan), PLANS);
+	  p->pln = pln;
+	  p->prb = prb;
+	  p->sign = sign; /* cache for execute_dft */
+	  
+	  /* blessing protocol */
+	  plnr->planner_flags |= BLESSING;
+	  pln0 = plnr->adt->mkplan(plnr, prb);
+	  X(plan_destroy_internal)(pln0);
+     } else {
+	  X(problem_destroy)(prb);
+     }
+     
+     /* discard all information not necessary to reconstruct the
+	plan */
+     plnr->adt->forget(plnr, FORGET_ACCURSED);
+     
+     return p;
+})
+
+void X(destroy_plan)(X(plan) p)
+{
+     if (p) {
+          AWAKE(p->pln, 0);
+          X(plan_destroy_internal)(p->pln);
+          X(problem_destroy)(p->prb);
+          X(ifree)(p);
+     }
+}
diff --git a/src/fftw3/api/config.h b/src/fftw3/api/config.h
new file mode 100644
index 0000000..e97bf0d
--- /dev/null
+++ b/src/fftw3/api/config.h
@@ -0,0 +1,301 @@
+/* configuration file for the IM_FFTW3 library */
+
+/* extra CFLAGS for codelets */
+#define CODELET_OPTIM	""
+
+/* Define to a macro mangling the given C identifier (in lower and upper
+   case), which must not contain underscores, for linking with Fortran. */
+#undef F77_FUNC
+
+/* As F77_FUNC, but for C identifiers containing underscores. */
+#undef F77_FUNC_
+
+/* Define if F77_FUNC and F77_FUNC_ are equivalent. */
+#undef F77_FUNC_EQUIV
+
+/* C compiler name and flags */
+#define FFTW_CC	""
+
+/* Define to enable extra FFTW debugging code. */
+#undef FFTW_DEBUG
+
+/* Define to enable alignment debugging hacks. */
+#undef FFTW_DEBUG_ALIGNMENT
+
+/* Define to enable debugging malloc. */
+#undef FFTW_DEBUG_MALLOC
+
+/* enable fast, unsafe modular multiplications, risking overflow for large
+   prime sizes */
+#undef FFTW_ENABLE_UNSAFE_MULMOD
+
+/* Define to compile in long-double precision. */
+#undef FFTW_LDOUBLE
+
+/* Define to compile in single precision. */
+#define FFTW_SINGLE 1
+
+/* Define to enable 3DNow! optimizations. */
+#undef HAVE_3DNOW
+
+/* Define to 1 if you have `alloca', as a function or macro. */
+#undef HAVE_ALLOCA
+
+/* Define to 1 if you have <alloca.h> and it should be used (not on Ultrix).
+   */
+#undef HAVE_ALLOCA_H
+
+/* Define to enable Altivec optimizations. */
+#undef HAVE_ALTIVEC
+
+/* Define to 1 if you have the `BSDgettimeofday' function. */
+#undef HAVE_BSDGETTIMEOFDAY
+
+/* Define to 1 if you have the `clock_gettime' function. */
+#undef HAVE_CLOCK_GETTIME
+
+/* Define to 1 if you have the `cosl' function. */
+#undef HAVE_COSL
+
+/* Define to 1 if you have the <c_asm.h> header file. */
+#undef HAVE_C_ASM_H
+
+/* Define to 1 if you have the declaration of `drand48', and to 0 if you don't. */
+#define HAVE_DECL_DRAND48	0
+
+/* Define to 1 if you have the declaration of `memalign', and to 0 if you
+   don't. */
+#define HAVE_DECL_MEMALIGN	0
+
+/* Define to 1 if you have the declaration of `posix_memalign', and to 0 if
+   you don't. */
+#define HAVE_DECL_POSIX_MEMALIGN	0
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#undef HAVE_DLFCN_H
+
+/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */
+#undef HAVE_DOPRNT
+
+/* Define to 1 if you have the `drand48' function. */
+#undef HAVE_DRAND48
+
+/* Define to 1 if you have the `gethrtime' function. */
+#undef HAVE_GETHRTIME
+
+/* Define to 1 if you have the `gettimeofday' function. */
+#undef HAVE_GETTIMEOFDAY
+
+/* Define to 1 if hrtime_t is defined in <sys/time.h> */
+#undef HAVE_HRTIME_T
+
+/* Define to 1 if you have the <intrinsics.h> header file. */
+#undef HAVE_INTRINSICS_H
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
+
+/* Define if the isnan() function/macro is available. */
+#define HAVE_ISNAN	1
+
+/* Define to enable AMD K7 optimizations. */
+#undef HAVE_K7
+
+/* Define to 1 if you have the <libintl.h> header file. */
+#undef HAVE_LIBINTL_H
+
+/* Define to 1 if you have the `m' library (-lm). */
+#undef HAVE_LIBM
+
+/* Define to 1 if you have the <limits.h> header file. */
+#define HAVE_LIMITS_H	1
+
+/* Define to 1 if you have the <malloc.h> header file. */
+/* #define HAVE_MALLOC_H	1 */
+
+/* Define to 1 if you have the `memalign' function. */
+#undef HAVE_MEMALIGN
+
+/* Define to 1 if you have the <memory.h> header file. */
+#define HAVE_MEMORY_H	1
+
+/* Define to 1 if you have the `memset' function. */
+#define HAVE_MEMSET	1
+
+/* Define to 1 if you have the `posix_memalign' function. */
+#undef HAVE_POSIX_MEMALIGN
+
+/* Define if you have POSIX threads libraries and header files. */
+#undef HAVE_PTHREAD
+
+/* Define to 1 if you have the `read_real_time' function. */
+#undef HAVE_READ_REAL_TIME
+
+/* Define to 1 if you have the `sinl' function. */
+#define HAVE_SINL	1
+
+/* Define to 1 if you have the `sqrt' function. */
+#define HAVE_SQRT	1
+
+/* Define to enable SSE optimizations. */
+#undef HAVE_SSE
+/*#define HAVE_SSE 1 */
+
+/* Define to enable SSE2 optimizations. */
+#undef HAVE_SSE2
+
+/* Define to 1 if you have the <stddef.h> header file. */
+#define HAVE_STDDEF_H	1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#define HAVE_STDLIB_H	1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#define HAVE_STRING_H	1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#define HAVE_SYS_STAT_H	1
+
+/* Define to 1 if you have the <sys/time.h> header file. */
+#undef HAVE_SYS_TIME_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#define HAVE_SYS_TYPES_H	1
+
+/* Define to 1 if you have the `tanl' function. */
+#define HAVE_TANL	1
+
+/* Define if we have a threads library. */
+#undef HAVE_THREADS
+
+/* Define to 1 if you have the `time_base_to_time' function. */
+#undef HAVE_TIME_BASE_TO_TIME
+
+/* Define to 1 if the system has the type `uintptr_t'. */
+/* #undef HAVE_UINTPTR_T */
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* Define to 1 if you have the `vprintf' function. */
+#define HAVE_VPRINTF
+
+/* Define to 1 if you have the `_mm_free' function. */
+#undef HAVE__MM_FREE
+
+/* Define to 1 if you have the `_mm_malloc' function. */
+#undef HAVE__MM_MALLOC
+
+/* Define if you have the UNICOS _rtc() intrinsic. */
+#undef HAVE__RTC
+
+/* Name of package */
+#define PACKAGE	"FFTW"
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME	"FFTW_PACKAGENAME"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING	"FFTW V3"
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION	3
+
+/* Define to the necessary symbol if this constant uses a non-standard name on
+   your system. */
+#undef PTHREAD_CREATE_JOINABLE
+
+/* The size of a `double', as computed by sizeof. */
+#define SIZEOF_DOUBLE	8
+
+/* The size of a `int', as computed by sizeof. */
+#define SIZEOF_INT	4
+
+/* The size of a `long', as computed by sizeof. */
+#define SIZEOF_LONG	4
+
+/* The size of a `long double', as computed by sizeof. */
+#define SIZEOF_LONG_DOUBLE	8
+
+/* The size of a `long long', as computed by sizeof. */
+#undef SIZEOF_LONG_LONG	/* NO LONG LONG IN VC++ */
+
+/* The size of a `unsigned int', as computed by sizeof. */
+#define SIZEOF_UNSIGNED_INT	4
+
+/* The size of a `unsigned long', as computed by sizeof. */
+#define SIZEOF_UNSIGNED_LONG	4
+
+/* The size of a `unsigned long long', as computed by sizeof. */
+#undef SIZEOF_UNSIGNED_LONG_LONG
+
+/* The size of a `void *', as computed by sizeof. */
+#define SIZEOF_VOID_P	4
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+        STACK_DIRECTION > 0 => grows toward higher addresses
+        STACK_DIRECTION < 0 => grows toward lower addresses
+        STACK_DIRECTION = 0 => direction of growth unknown */
+#undef STACK_DIRECTION
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS	1
+
+/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
+#undef TIME_WITH_SYS_TIME
+
+/* Define if we have and are using OpenMP multithreading directives */
+#undef USING_OPENMP_THREADS
+
+/* Define if we have and are using POSIX threads. */
+#undef USING_POSIX_THREADS
+
+/* Version number of package */
+#define VERSION	"3"
+
+/* Use common Windows Fortran mangling styles for the Fortran interfaces. */
+#undef WINDOWS_F77_MANGLING
+
+/* Use our own 16-byte aligned malloc routine; mainly helpful for Windows
+   systems lacking aligned allocation system-library routines. */
+#undef WITHOUT_CYCLE_COUNTER
+
+/* Use our own 16-byte aligned malloc routine; mainly helpful for Windows
+   systems lacking aligned allocation system-library routines. */
+#define WITH_OUR_MALLOC16	1
+
+/* Define to empty if `const' does not conform to ANSI C. */
+#undef const
+
+/* Define as `__inline' if that's what the C compiler calls it, or to nothing
+   if it is not supported. */
+#define inline	__inline
+
+/* Define to `unsigned' if <sys/types.h> does not define. */
+#undef size_t
+
+/*			 VC++ specific parameters  */
+/*			added May14th 2003, S.Ruel */
+#define		HAVE_CONFIG_H		1
+#define		isnan				_isnan
+#define		__inline__			__inline
+#define		__asm__				__asm
+
+/* The intel and processor pack libraries should be compatible */
+#ifdef _MSC_VER
+#define		__ICC				1
+#endif
diff --git a/src/fftw3/api/configure.c b/src/fftw3/api/configure.c
new file mode 100644
index 0000000..995281b
--- /dev/null
+++ b/src/fftw3/api/configure.c
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "dft.h"
+#include "rdft.h"
+#include "reodft.h"
+
+void X(configure_planner)(planner *plnr)
+{
+     X(dft_conf_standard)(plnr);
+     X(rdft_conf_standard)(plnr);
+     X(reodft_conf_standard)(plnr);
+}
diff --git a/src/fftw3/api/execute-dft-c2r.c b/src/fftw3/api/execute-dft-c2r.c
new file mode 100644
index 0000000..18b0749
--- /dev/null
+++ b/src/fftw3/api/execute-dft-c2r.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+/* guru interface: requires care in alignment, r - i, etcetera. */
+void X(execute_dft_c2r)(const X(plan) p, C *in, R *out)
+WITH_ALIGNED_STACK({
+     plan_rdft2 *pln = (plan_rdft2 *) p->pln;
+     pln->apply((plan *) pln, out, in[0], in[0]+1);
+})
diff --git a/src/fftw3/api/execute-dft-r2c.c b/src/fftw3/api/execute-dft-r2c.c
new file mode 100644
index 0000000..1f4818f
--- /dev/null
+++ b/src/fftw3/api/execute-dft-r2c.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+/* guru interface: requires care in alignment, r - i, etcetera. */
+void X(execute_dft_r2c)(const X(plan) p, R *in, C *out)
+WITH_ALIGNED_STACK({
+     plan_rdft2 *pln = (plan_rdft2 *) p->pln;
+     pln->apply((plan *) pln, in, out[0], out[0]+1);
+})
diff --git a/src/fftw3/api/execute-dft.c b/src/fftw3/api/execute-dft.c
new file mode 100644
index 0000000..67fc9b8
--- /dev/null
+++ b/src/fftw3/api/execute-dft.c
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "dft.h"
+
+/* guru interface: requires care in alignment etcetera. */
+void X(execute_dft)(const X(plan) p, C *in, C *out)
+WITH_ALIGNED_STACK({
+     plan_dft *pln = (plan_dft *) p->pln;
+     if (p->sign == FFT_SIGN)
+	  pln->apply((plan *) pln, in[0], in[0]+1, out[0], out[0]+1);
+     else
+	  pln->apply((plan *) pln, in[0]+1, in[0], out[0]+1, out[0]);
+})
diff --git a/src/fftw3/api/execute-r2r.c b/src/fftw3/api/execute-r2r.c
new file mode 100644
index 0000000..d05f4ef
--- /dev/null
+++ b/src/fftw3/api/execute-r2r.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+/* guru interface: requires care in alignment, etcetera. */
+void X(execute_r2r)(const X(plan) p, R *in, R *out)
+WITH_ALIGNED_STACK({
+     plan_rdft *pln = (plan_rdft *) p->pln;
+     pln->apply((plan *) pln, in, out);
+})
diff --git a/src/fftw3/api/execute-split-dft-c2r.c b/src/fftw3/api/execute-split-dft-c2r.c
new file mode 100644
index 0000000..5941436
--- /dev/null
+++ b/src/fftw3/api/execute-split-dft-c2r.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+/* guru interface: requires care in alignment, r - i, etcetera. */
+void X(execute_split_dft_c2r)(const X(plan) p, R *ri, R *ii, R *out)
+WITH_ALIGNED_STACK({
+     plan_rdft2 *pln = (plan_rdft2 *) p->pln;
+     pln->apply((plan *) pln, out, ri, ii);
+})
diff --git a/src/fftw3/api/execute-split-dft-r2c.c b/src/fftw3/api/execute-split-dft-r2c.c
new file mode 100644
index 0000000..72eccb8
--- /dev/null
+++ b/src/fftw3/api/execute-split-dft-r2c.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+/* guru interface: requires care in alignment, r - i, etcetera. */
+void X(execute_split_dft_r2c)(const X(plan) p, R *in, R *ro, R *io)
+WITH_ALIGNED_STACK({
+     plan_rdft2 *pln = (plan_rdft2 *) p->pln;
+     pln->apply((plan *) pln, in, ro, io);
+})
diff --git a/src/fftw3/api/execute-split-dft.c b/src/fftw3/api/execute-split-dft.c
new file mode 100644
index 0000000..4c226a9
--- /dev/null
+++ b/src/fftw3/api/execute-split-dft.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "dft.h"
+
+/* guru interface: requires care in alignment, r - i, etcetera. */
+void X(execute_split_dft)(const X(plan) p, R *ri, R *ii, R *ro, R *io)
+WITH_ALIGNED_STACK({
+     plan_dft *pln = (plan_dft *) p->pln;
+     pln->apply((plan *) pln, ri, ii, ro, io);
+})
diff --git a/src/fftw3/api/execute.c b/src/fftw3/api/execute.c
new file mode 100644
index 0000000..f995c1c
--- /dev/null
+++ b/src/fftw3/api/execute.c
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+void X(execute)(const X(plan) p)
+WITH_ALIGNED_STACK({
+     plan *pln = p->pln;
+     pln->adt->solve(pln, p->prb);
+})
diff --git a/src/fftw3/api/export-wisdom-to-file.c b/src/fftw3/api/export-wisdom-to-file.c
new file mode 100644
index 0000000..769f1b4
--- /dev/null
+++ b/src/fftw3/api/export-wisdom-to-file.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+void X(export_wisdom_to_file)(FILE *output_file)
+{
+     printer *p = X(mkprinter_file)(output_file);
+     planner *plnr = X(the_planner)();
+     plnr->adt->exprt(plnr, p);
+     X(printer_destroy)(p);
+}
diff --git a/src/fftw3/api/export-wisdom-to-string.c b/src/fftw3/api/export-wisdom-to-string.c
new file mode 100644
index 0000000..df38b51
--- /dev/null
+++ b/src/fftw3/api/export-wisdom-to-string.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+typedef struct {
+     printer super;
+     int *cnt;
+} P_cnt;
+
+static void putchr_cnt(printer * p_, char c)
+{
+     P_cnt *p = (P_cnt *) p_;
+     UNUSED(c);
+     ++*p->cnt;
+}
+
+static printer *mkprinter_cnt(int *cnt)
+{
+     P_cnt *p = (P_cnt *) X(mkprinter)(sizeof(P_cnt), putchr_cnt, 0);
+     p->cnt = cnt;
+     *cnt = 0;
+     return &p->super;
+}
+
+typedef struct {
+     printer super;
+     char *s;
+} P_str;
+
+static void putchr_str(printer * p_, char c)
+{
+     P_str *p = (P_str *) p_;
+     *p->s++ = c;
+     *p->s = 0;
+}
+
+static printer *mkprinter_str(char *s)
+{
+     P_str *p = (P_str *) X(mkprinter)(sizeof(P_str), putchr_str, 0);
+     p->s = s;
+     *s = 0;
+     return &p->super;
+}
+
+char *X(export_wisdom_to_string)(void)
+{
+     printer *p;
+     planner *plnr = X(the_planner)();
+     int cnt;
+     char *s;
+
+     p = mkprinter_cnt(&cnt);
+     plnr->adt->exprt(plnr, p);
+     X(printer_destroy)(p);
+
+     s = (char *) NATIVE_MALLOC(sizeof(char) * (cnt + 1), OTHER);
+     if (s) {
+          p = mkprinter_str(s);
+          plnr->adt->exprt(plnr, p);
+          X(printer_destroy)(p);
+     }
+
+     return s;
+}
diff --git a/src/fftw3/api/export-wisdom.c b/src/fftw3/api/export-wisdom.c
new file mode 100644
index 0000000..d05c067
--- /dev/null
+++ b/src/fftw3/api/export-wisdom.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+typedef struct {
+     printer super;
+     void (*write_char)(char c, void *);
+     void *data;
+} P;
+
+static void putchr_generic(printer * p_, char c)
+{
+     P *p = (P *) p_;
+     (p->write_char)(c, p->data);
+}
+
+void X(export_wisdom)(void (*write_char)(char c, void *), void *data)
+{
+     P *p = (P *) X(mkprinter)(sizeof(P), putchr_generic, 0);
+     planner *plnr = X(the_planner)();
+
+     p->write_char = write_char;
+     p->data = data;
+     plnr->adt->exprt(plnr, (printer *) p);
+     X(printer_destroy)((printer *) p);
+}
diff --git a/src/fftw3/api/extract-reim.c b/src/fftw3/api/extract-reim.c
new file mode 100644
index 0000000..3a90ab5
--- /dev/null
+++ b/src/fftw3/api/extract-reim.c
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+/* decompose complex pointer into real and imaginary parts.
+   Flip real and imaginary if there the sign does not match
+   FFTW's idea of what the sign should be */
+
+void X(extract_reim)(int sign, C *c, R **r, R **i)
+{
+     if (sign == FFT_SIGN) {
+          *r = c[0] + 0;
+          *i = c[0] + 1;
+     } else {
+          *r = c[0] + 1;
+          *i = c[0] + 0;
+     }
+}
diff --git a/src/fftw3/api/f77api.c b/src/fftw3/api/f77api.c
new file mode 100644
index 0000000..3f70716
--- /dev/null
+++ b/src/fftw3/api/f77api.c
@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "dft.h"
+#include "rdft.h"
+
+/* if F77_FUNC is not defined, then we don't know how to mangle identifiers
+   for the Fortran linker, and we must omit the f77 API. */
+#if defined(F77_FUNC) || defined(WINDOWS_F77_MANGLING)
+
+/*-----------------------------------------------------------------------*/
+/* some internal functions used by the f77 api */
+
+/* in fortran, the natural array ordering is column-major, which
+   corresponds to reversing the dimensions relative to C's row-major */
+static int *reverse_n(int rnk, const int *n)
+{
+     int *nrev;
+     int i;
+     A(FINITE_RNK(rnk));
+     nrev = (int *) MALLOC(sizeof(int) * rnk, PROBLEMS);
+     for (i = 0; i < rnk; ++i)
+          nrev[rnk - i - 1] = n[i];
+     return nrev;
+}
+
+/* f77 doesn't have data structures, so we have to pass iodims as
+   parallel arrays */
+static X(iodim) *make_dims(int rnk, const int *n,
+			   const int *is, const int *os)
+{
+     X(iodim) *dims;
+     int i;
+     A(FINITE_RNK(rnk));
+     dims = (X(iodim) *) MALLOC(sizeof(X(iodim)) * rnk, PROBLEMS);
+     for (i = 0; i < rnk; ++i) {
+          dims[i].n = n[i];
+          dims[i].is = is[i];
+          dims[i].os = os[i];
+     }
+     return dims;
+}
+
+typedef struct {
+     void (*f77_write_char)(char *, void *);
+     void *data;
+} write_char_data;
+
+static void write_char(char c, void *d)
+{
+     write_char_data *ad = (write_char_data *) d;
+     ad->f77_write_char(&c, ad->data);
+}
+
+typedef struct {
+     void (*f77_read_char)(int *, void *);
+     void *data;
+} read_char_data;
+
+static int read_char(void *d)
+{
+     read_char_data *ed = (read_char_data *) d;
+     int c;
+     ed->f77_read_char(&c, ed->data);
+     return (c < 0 ? EOF : c);
+}
+
+static X(r2r_kind) *ints2kinds(int rnk, const int *ik)
+{
+     if (!FINITE_RNK(rnk) || rnk == 0)
+	  return 0;
+     else {
+	  int i;
+	  X(r2r_kind) *k;
+
+	  k = (X(r2r_kind) *) MALLOC(sizeof(X(r2r_kind)) * rnk, PROBLEMS);
+	  /* reverse order for Fortran -> C */
+	  for (i = 0; i < rnk; ++i)
+	       k[i] = (X(r2r_kind)) ik[rnk - 1 - i];
+	  return k;
+     }
+}
+
+/*-----------------------------------------------------------------------*/
+
+#include "x77.h"
+
+#define F77(a, A) F77x(x77(a), X77(A))
+
+#ifndef WINDOWS_F77_MANGLING
+
+#if defined(F77_FUNC)
+#  define F77x(a, A) F77_FUNC(a, A)
+#  include "f77funcs.h"
+#endif
+
+/* If identifiers with underscores are mangled differently than those
+   without underscores, then we include *both* mangling versions.  The
+   reason is that the only Fortran compiler that does such differing
+   mangling is currently g77 (which adds an extra underscore to names
+   with underscores), whereas other compilers running on the same
+   machine are likely to use g77's non-underscored mangling.  (I'm sick
+   of users complaining that FFTW works with g77 but not with e.g.
+   pgf77 or ifc on the same machine.)  Note that all FFTW identifiers
+   contain underscores, and configure picks g77 by default. */
+#if defined(F77_FUNC_) && !defined(F77_FUNC_EQUIV)
+#  undef F77x
+#  define F77x(a, A) F77_FUNC_(a, A)
+#  include "f77funcs.h"
+#endif
+
+#else /* WINDOWS_F77_MANGLING */
+
+/* Various mangling conventions common (?) under Windows. */
+
+/* g77 */
+#  define WINDOWS_F77_FUNC(a, A) a ## __
+#  define F77x(a, A) WINDOWS_F77_FUNC(a, A)
+#  include "f77funcs.h"
+
+/* Digital/Compaq/HP Visual Fortran, Intel Fortran.  stdcall attribute
+   is apparently required to adjust for calling conventions (callee
+   pops stack in stdcall).  See also:
+       http://msdn.microsoft.com/library/en-us/vccore98/html/_core_mixed.2d.language_programming.3a_.overview.asp
+*/
+#  undef WINDOWS_F77_FUNC
+#  if defined(__GNUC__)
+#    define WINDOWS_F77_FUNC(a, A) __attribute__((stdcall)) A
+#  elif defined(_MSC_VER) || defined(_ICC) || defined(_STDCALL_SUPPORTED)
+#    define WINDOWS_F77_FUNC(a, A) __stdcall A
+#  else
+#    define WINDOWS_F77_FUNC(a, A) A /* oh well */
+#  endif
+#  include "f77funcs.h"
+
+#endif /* WINDOWS_F77_MANGLING */
+
+#endif				/* F77_FUNC */
diff --git a/src/fftw3/api/f77funcs.h b/src/fftw3/api/f77funcs.h
new file mode 100644
index 0000000..ea0dd5c
--- /dev/null
+++ b/src/fftw3/api/f77funcs.h
@@ -0,0 +1,438 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* Functions in the FFTW Fortran API, mangled according to the
+   F77(...) macro.  This file is designed to be #included by
+   f77api.c, possibly multiple times in order to support multiple
+   compiler manglings (via redefinition of F77). */
+
+void F77(execute, EXECUTE)(X(plan) * const p)
+WITH_ALIGNED_STACK({
+     plan *pln = (*p)->pln;
+     pln->adt->solve(pln, (*p)->prb);
+})
+
+void F77(destroy_plan, DESTROY_PLAN)(X(plan) *p)
+{
+     X(destroy_plan)(*p);
+}
+
+void F77(cleanup, CLEANUP)(void)
+{
+     X(cleanup)();
+}
+
+void F77(forget_wisdom, FORGET_WISDOM)(void)
+{
+     X(forget_wisdom)();
+}
+
+void F77(export_wisdom, EXPORT_WISDOM)(void (*f77_write_char)(char *, void *),
+				       void *data)
+{
+     write_char_data ad;
+     ad.f77_write_char = f77_write_char;
+     ad.data = data;
+     X(export_wisdom)(write_char, (void *) &ad);
+}
+
+void F77(import_wisdom, IMPORT_WISDOM)(int *isuccess,
+				       void (*f77_read_char)(int *, void *),
+				       void *data)
+{
+     read_char_data ed;
+     ed.f77_read_char = f77_read_char;
+     ed.data = data;
+     *isuccess = X(import_wisdom)(read_char, (void *) &ed);
+}
+
+void F77(import_system_wisdom, IMPORT_SYSTEM_WISDOM)(int *isuccess)
+{
+     *isuccess = X(import_system_wisdom)();
+}
+
+void F77(print_plan, PRINT_PLAN)(X(plan) * const p)
+{
+     X(print_plan)(*p);
+}
+
+void F77(flops,FLOPS)(X(plan) *p, double *add, double *mul, double *fma)
+{
+     X(flops)(*p, add, mul, fma);
+}
+
+/******************************** DFT ***********************************/
+
+void F77(plan_dft, PLAN_DFT)(X(plan) *p, int *rank, const int *n,
+			     C *in, C *out, int *sign, int *flags)
+{
+     int *nrev = reverse_n(*rank, n);
+     *p = X(plan_dft)(*rank, nrev, in, out, *sign, *flags);
+     X(ifree0)(nrev);
+}
+
+void F77(plan_dft_1d, PLAN_DFT_1D)(X(plan) *p, int *n, C *in, C *out,
+				   int *sign, int *flags)
+{
+     *p = X(plan_dft_1d)(*n, in, out, *sign, *flags);
+}
+
+void F77(plan_dft_2d, PLAN_DFT_2D)(X(plan) *p, int *nx, int *ny,
+				   C *in, C *out, int *sign, int *flags)
+{
+     *p = X(plan_dft_2d)(*ny, *nx, in, out, *sign, *flags);
+}
+
+void F77(plan_dft_3d, PLAN_DFT_3D)(X(plan) *p, int *nx, int *ny, int *nz,
+				   C *in, C *out,
+				   int *sign, int *flags)
+{
+     *p = X(plan_dft_3d)(*nz, *ny, *nx, in, out, *sign, *flags);
+}
+
+void F77(plan_many_dft, PLAN_MANY_DFT)(X(plan) *p, int *rank, const int *n,
+				       int *howmany,
+				       C *in, const int *inembed,
+				       int *istride, int *idist,
+				       C *out, const int *onembed,
+				       int *ostride, int *odist,
+				       int *sign, int *flags)
+{
+     int *nrev = reverse_n(*rank, n);
+     int *inembedrev = reverse_n(*rank, inembed);
+     int *onembedrev = reverse_n(*rank, onembed);
+     *p = X(plan_many_dft)(*rank, nrev, *howmany,
+			   in, inembedrev, *istride, *idist,
+			   out, onembedrev, *ostride, *odist,
+			   *sign, *flags);
+     X(ifree0)(onembedrev);
+     X(ifree0)(inembedrev);
+     X(ifree0)(nrev);
+}
+
+void F77(plan_guru_dft, PLAN_GURU_DFT)(X(plan) *p, int *rank, const int *n,
+				       const int *is, const int *os,
+				       int *howmany_rank, const int *h_n,
+				       const int *h_is, const int *h_os,
+				       C *in, C *out, int *sign, int *flags)
+{
+     X(iodim) *dims = make_dims(*rank, n, is, os);
+     X(iodim) *howmany_dims = make_dims(*howmany_rank, h_n, h_is, h_os);
+     *p = X(plan_guru_dft)(*rank, dims, *howmany_rank, howmany_dims,
+			   in, out, *sign, *flags);
+     X(ifree0)(howmany_dims);
+     X(ifree0)(dims);
+}
+
+void F77(plan_guru_split_dft, PLAN_GURU_SPLIT_DFT)(X(plan) *p, int *rank, const int *n,
+				       const int *is, const int *os,
+				       int *howmany_rank, const int *h_n,
+				       const int *h_is, const int *h_os,
+				       R *ri, R *ii, R *ro, R *io, int *flags)
+{
+     X(iodim) *dims = make_dims(*rank, n, is, os);
+     X(iodim) *howmany_dims = make_dims(*howmany_rank, h_n, h_is, h_os);
+     *p = X(plan_guru_split_dft)(*rank, dims, *howmany_rank, howmany_dims,
+			   ri, ii, ro, io, *flags);
+     X(ifree0)(howmany_dims);
+     X(ifree0)(dims);
+}
+
+void F77(execute_dft, EXECUTE_DFT)(X(plan) * const p, C *in, C *out)
+WITH_ALIGNED_STACK({
+     plan_dft *pln = (plan_dft *) (*p)->pln;
+     if ((*p)->sign == FFT_SIGN)
+          pln->apply((plan *) pln, in[0], in[0]+1, out[0], out[0]+1);
+     else
+          pln->apply((plan *) pln, in[0]+1, in[0], out[0]+1, out[0]);
+})
+
+void F77(execute_split_dft, EXECUTE_SPLIT_DFT)(X(plan) * const p,
+					       R *ri, R *ii, R *ro, R *io)
+WITH_ALIGNED_STACK({
+     plan_dft *pln = (plan_dft *) (*p)->pln;
+     pln->apply((plan *) pln, ri, ii, ro, io);
+})
+
+/****************************** DFT r2c *********************************/
+
+void F77(plan_dft_r2c, PLAN_DFT_R2C)(X(plan) *p, int *rank, const int *n,
+				     R *in, C *out, int *flags)
+{
+     int *nrev = reverse_n(*rank, n);
+     *p = X(plan_dft_r2c)(*rank, nrev, in, out, *flags);
+     X(ifree0)(nrev);
+}
+
+void F77(plan_dft_r2c_1d, PLAN_DFT_R2C_1D)(X(plan) *p, int *n, R *in, C *out,
+					   int *flags)
+{
+     *p = X(plan_dft_r2c_1d)(*n, in, out, *flags);
+}
+
+void F77(plan_dft_r2c_2d, PLAN_DFT_R2C_2D)(X(plan) *p, int *nx, int *ny,
+					   R *in, C *out, int *flags)
+{
+     *p = X(plan_dft_r2c_2d)(*ny, *nx, in, out, *flags);
+}
+
+void F77(plan_dft_r2c_3d, PLAN_DFT_R2C_3D)(X(plan) *p,
+					   int *nx, int *ny, int *nz,
+					   R *in, C *out,
+					   int *flags)
+{
+     *p = X(plan_dft_r2c_3d)(*nz, *ny, *nx, in, out, *flags);
+}
+
+void F77(plan_many_dft_r2c, PLAN_MANY_DFT_R2C)(
+     X(plan) *p, int *rank, const int *n,
+     int *howmany,
+     R *in, const int *inembed, int *istride, int *idist,
+     C *out, const int *onembed, int *ostride, int *odist,
+     int *flags)
+{
+     int *nrev = reverse_n(*rank, n);
+     int *inembedrev = reverse_n(*rank, inembed);
+     int *onembedrev = reverse_n(*rank, onembed);
+     *p = X(plan_many_dft_r2c)(*rank, nrev, *howmany,
+			       in, inembedrev, *istride, *idist,
+			       out, onembedrev, *ostride, *odist,
+			       *flags);
+     X(ifree0)(onembedrev);
+     X(ifree0)(inembedrev);
+     X(ifree0)(nrev);
+}
+
+void F77(plan_guru_dft_r2c, PLAN_GURU_DFT_R2C)(
+     X(plan) *p, int *rank, const int *n,
+     const int *is, const int *os,
+     int *howmany_rank, const int *h_n,
+     const int *h_is, const int *h_os,
+     R *in, C *out, int *flags)
+{
+     X(iodim) *dims = make_dims(*rank, n, is, os);
+     X(iodim) *howmany_dims = make_dims(*howmany_rank, h_n, h_is, h_os);
+     *p = X(plan_guru_dft_r2c)(*rank, dims, *howmany_rank, howmany_dims,
+			       in, out, *flags);
+     X(ifree0)(howmany_dims);
+     X(ifree0)(dims);
+}
+
+void F77(plan_guru_split_dft_r2c, PLAN_GURU_SPLIT_DFT_R2C)(
+     X(plan) *p, int *rank, const int *n,
+     const int *is, const int *os,
+     int *howmany_rank, const int *h_n,
+     const int *h_is, const int *h_os,
+     R *in, R *ro, R *io, int *flags)
+{
+     X(iodim) *dims = make_dims(*rank, n, is, os);
+     X(iodim) *howmany_dims = make_dims(*howmany_rank, h_n, h_is, h_os);
+     *p = X(plan_guru_split_dft_r2c)(*rank, dims, *howmany_rank, howmany_dims,
+			       in, ro, io, *flags);
+     X(ifree0)(howmany_dims);
+     X(ifree0)(dims);
+}
+
+void F77(execute_dft_r2c, EXECUTE_DFT_R2C)(X(plan) * const p, R *in, C *out)
+WITH_ALIGNED_STACK({
+     plan_rdft2 *pln = (plan_rdft2 *) (*p)->pln;
+     pln->apply((plan *) pln, in, out[0], out[0]+1);
+})
+
+void F77(execute_split_dft_r2c, EXECUTE_SPLIT_DFT_R2C)(X(plan) * const p,
+						       R *in, R *ro, R *io)
+WITH_ALIGNED_STACK({
+     plan_rdft2 *pln = (plan_rdft2 *) (*p)->pln;
+     pln->apply((plan *) pln, in, ro, io);
+})
+
+/****************************** DFT c2r *********************************/
+
+void F77(plan_dft_c2r, PLAN_DFT_C2R)(X(plan) *p, int *rank, const int *n,
+				     C *in, R *out, int *flags)
+{
+     int *nrev = reverse_n(*rank, n);
+     *p = X(plan_dft_c2r)(*rank, nrev, in, out, *flags);
+     X(ifree0)(nrev);
+}
+
+void F77(plan_dft_c2r_1d, PLAN_DFT_C2R_1D)(X(plan) *p, int *n, C *in, R *out,
+					   int *flags)
+{
+     *p = X(plan_dft_c2r_1d)(*n, in, out, *flags);
+}
+
+void F77(plan_dft_c2r_2d, PLAN_DFT_C2R_2D)(X(plan) *p, int *nx, int *ny,
+					   C *in, R *out, int *flags)
+{
+     *p = X(plan_dft_c2r_2d)(*ny, *nx, in, out, *flags);
+}
+
+void F77(plan_dft_c2r_3d, PLAN_DFT_C2R_3D)(X(plan) *p,
+					   int *nx, int *ny, int *nz,
+					   C *in, R *out,
+					   int *flags)
+{
+     *p = X(plan_dft_c2r_3d)(*nz, *ny, *nx, in, out, *flags);
+}
+
+void F77(plan_many_dft_c2r, PLAN_MANY_DFT_C2R)(
+     X(plan) *p, int *rank, const int *n,
+     int *howmany,
+     C *in, const int *inembed, int *istride, int *idist,
+     R *out, const int *onembed, int *ostride, int *odist,
+     int *flags)
+{
+     int *nrev = reverse_n(*rank, n);
+     int *inembedrev = reverse_n(*rank, inembed);
+     int *onembedrev = reverse_n(*rank, onembed);
+     *p = X(plan_many_dft_c2r)(*rank, nrev, *howmany,
+			       in, inembedrev, *istride, *idist,
+			       out, onembedrev, *ostride, *odist,
+			       *flags);
+     X(ifree0)(onembedrev);
+     X(ifree0)(inembedrev);
+     X(ifree0)(nrev);
+}
+
+void F77(plan_guru_dft_c2r, PLAN_GURU_DFT_C2R)(
+     X(plan) *p, int *rank, const int *n,
+     const int *is, const int *os,
+     int *howmany_rank, const int *h_n,
+     const int *h_is, const int *h_os,
+     C *in, R *out, int *flags)
+{
+     X(iodim) *dims = make_dims(*rank, n, is, os);
+     X(iodim) *howmany_dims = make_dims(*howmany_rank, h_n, h_is, h_os);
+     *p = X(plan_guru_dft_c2r)(*rank, dims, *howmany_rank, howmany_dims,
+			       in, out, *flags);
+     X(ifree0)(howmany_dims);
+     X(ifree0)(dims);
+}
+
+void F77(plan_guru_split_dft_c2r, PLAN_GURU_SPLIT_DFT_C2R)(
+     X(plan) *p, int *rank, const int *n,
+     const int *is, const int *os,
+     int *howmany_rank, const int *h_n,
+     const int *h_is, const int *h_os,
+     R *ri, R *ii, R *out, int *flags)
+{
+     X(iodim) *dims = make_dims(*rank, n, is, os);
+     X(iodim) *howmany_dims = make_dims(*howmany_rank, h_n, h_is, h_os);
+     *p = X(plan_guru_split_dft_c2r)(*rank, dims, *howmany_rank, howmany_dims,
+			       ri, ii, out, *flags);
+     X(ifree0)(howmany_dims);
+     X(ifree0)(dims);
+}
+
+void F77(execute_dft_c2r, EXECUTE_DFT_C2R)(X(plan) * const p, C *in, R *out)
+WITH_ALIGNED_STACK({
+     plan_rdft2 *pln = (plan_rdft2 *) (*p)->pln;
+     pln->apply((plan *) pln, out, in[0], in[0]+1);
+})
+
+void F77(execute_split_dft_c2r, EXECUTE_SPLIT_DFT_C2R)(X(plan) * const p,
+					   R *ri, R *ii, R *out)
+WITH_ALIGNED_STACK({
+     plan_rdft2 *pln = (plan_rdft2 *) (*p)->pln;
+     pln->apply((plan *) pln, out, ri, ii);
+})
+
+/****************************** r2r *********************************/
+
+void F77(plan_r2r, PLAN_R2R)(X(plan) *p, int *rank, const int *n,
+			     R *in, R *out,
+			     int *kind, int *flags)
+{
+     int *nrev = reverse_n(*rank, n);
+     X(r2r_kind) *k = ints2kinds(*rank, kind);
+     *p = X(plan_r2r)(*rank, nrev, in, out, k, *flags);
+     X(ifree0)(k);
+     X(ifree0)(nrev);
+}
+
+void F77(plan_r2r_1d, PLAN_R2R_1D)(X(plan) *p, int *n, R *in, R *out,
+				   int *kind, int *flags)
+{
+     *p = X(plan_r2r_1d)(*n, in, out, (X(r2r_kind)) *kind, *flags);
+}
+
+void F77(plan_r2r_2d, PLAN_R2R_2D)(X(plan) *p, int *nx, int *ny,
+				   R *in, R *out, 
+				   int *kindx, int *kindy, int *flags)
+{
+     *p = X(plan_r2r_2d)(*ny, *nx, in, out,
+			 (X(r2r_kind)) *kindy, (X(r2r_kind)) *kindx, *flags);
+}
+
+void F77(plan_r2r_3d, PLAN_R2R_3D)(X(plan) *p,
+				   int *nx, int *ny, int *nz,
+				   R *in, R *out,
+				   int *kindx, int *kindy, int *kindz,
+				   int *flags)
+{
+     *p = X(plan_r2r_3d)(*nz, *ny, *nx, in, out,
+			 (X(r2r_kind)) *kindz, (X(r2r_kind)) *kindy, 
+			 (X(r2r_kind)) *kindx, *flags);
+}
+
+void F77(plan_many_r2r, PLAN_MANY_R2R)(
+     X(plan) *p, int *rank, const int *n,
+     int *howmany,
+     R *in, const int *inembed, int *istride, int *idist,
+     R *out, const int *onembed, int *ostride, int *odist,
+     int *kind, int *flags)
+{
+     int *nrev = reverse_n(*rank, n);
+     int *inembedrev = reverse_n(*rank, inembed);
+     int *onembedrev = reverse_n(*rank, onembed);
+     X(r2r_kind) *k = ints2kinds(*rank, kind);
+     *p = X(plan_many_r2r)(*rank, nrev, *howmany,
+			       in, inembedrev, *istride, *idist,
+			       out, onembedrev, *ostride, *odist,
+			       k, *flags);
+     X(ifree0)(k);
+     X(ifree0)(onembedrev);
+     X(ifree0)(inembedrev);
+     X(ifree0)(nrev);
+}
+
+void F77(plan_guru_r2r, PLAN_GURU_R2R)(
+     X(plan) *p, int *rank, const int *n,
+     const int *is, const int *os,
+     int *howmany_rank, const int *h_n,
+     const int *h_is, const int *h_os,
+     R *in, R *out, int *kind, int *flags)
+{
+     X(iodim) *dims = make_dims(*rank, n, is, os);
+     X(iodim) *howmany_dims = make_dims(*howmany_rank, h_n, h_is, h_os);
+     X(r2r_kind) *k = ints2kinds(*rank, kind);
+     *p = X(plan_guru_r2r)(*rank, dims, *howmany_rank, howmany_dims,
+			       in, out, k, *flags);
+     X(ifree0)(k);
+     X(ifree0)(howmany_dims);
+     X(ifree0)(dims);
+}
+
+void F77(execute_r2r, EXECUTE_R2R)(X(plan) * const p, R *in, R *out)
+WITH_ALIGNED_STACK({
+     plan_rdft *pln = (plan_rdft *) (*p)->pln;
+     pln->apply((plan *) pln, in, out);
+})
diff --git a/src/fftw3/api/fftw3.h b/src/fftw3/api/fftw3.h
new file mode 100644
index 0000000..6cb3254
--- /dev/null
+++ b/src/fftw3/api/fftw3.h
@@ -0,0 +1,272 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* header file for fftw3 */
+/* $Id: fftw3.h,v 1.1 2008/10/17 06:10:47 scuri Exp $ */
+
+#ifndef FFTW3_H
+#define FFTW3_H
+
+#include <stdio.h>
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif /* __cplusplus */
+
+/* If <complex.h> is included, use the C99 complex type.  Otherwise
+   define a type bit-compatible with C99 complex */
+#ifdef _Complex_I
+#  define FFTW_DEFINE_COMPLEX(R, C) typedef R _Complex C
+#else
+#  define FFTW_DEFINE_COMPLEX(R, C) typedef R C[2]
+#endif
+
+#define FFTW_CONCAT(prefix, name) prefix ## name
+#define FFTW_MANGLE_DOUBLE(name) FFTW_CONCAT(fftw_, name)
+#define FFTW_MANGLE_FLOAT(name) FFTW_CONCAT(fftwf_, name)
+#define FFTW_MANGLE_LONG_DOUBLE(name) FFTW_CONCAT(fftwl_, name)
+
+
+enum fftw_r2r_kind_do_not_use_me {
+     FFTW_R2HC=0, FFTW_HC2R=1, FFTW_DHT=2,
+     FFTW_REDFT00=3, FFTW_REDFT01=4, FFTW_REDFT10=5, FFTW_REDFT11=6,
+     FFTW_RODFT00=7, FFTW_RODFT01=8, FFTW_RODFT10=9, FFTW_RODFT11=10
+};
+
+struct fftw_iodim_do_not_use_me {
+     int n;                     /* dimension size */
+     int is;			/* input stride */
+     int os;			/* output stride */
+};
+
+/*
+  huge second-order macro that defines prototypes for all API
+  functions.  We expand this macro for each supported precision
+ 
+  X: name-mangling macro
+  R: real data type
+  C: complex data type
+*/
+
+#define FFTW_DEFINE_API(X, R, C)					\
+									\
+FFTW_DEFINE_COMPLEX(R, C);						\
+									\
+typedef struct X(plan_s) *X(plan);					\
+									\
+typedef struct fftw_iodim_do_not_use_me X(iodim);			\
+									\
+typedef enum fftw_r2r_kind_do_not_use_me X(r2r_kind);			\
+									\
+void X(execute)(const X(plan) p);					\
+									\
+X(plan) X(plan_dft)(int rank, const int *n,				\
+		    C *in, C *out, int sign, unsigned flags);		\
+									\
+X(plan) X(plan_dft_1d)(int n, C *in, C *out, int sign,			\
+		       unsigned flags);					\
+X(plan) X(plan_dft_2d)(int nx, int ny,					\
+		       C *in, C *out, int sign, unsigned flags);	\
+X(plan) X(plan_dft_3d)(int nx, int ny, int nz,				\
+		       C *in, C *out, int sign, unsigned flags);	\
+									\
+X(plan) X(plan_many_dft)(int rank, const int *n,			\
+                         int howmany,					\
+                         C *in, const int *inembed,			\
+                         int istride, int idist,			\
+                         C *out, const int *onembed,			\
+                         int ostride, int odist,			\
+                         int sign, unsigned flags);			\
+									\
+X(plan) X(plan_guru_dft)(int rank, const X(iodim) *dims,		\
+			 int howmany_rank,				\
+			 const X(iodim) *howmany_dims,			\
+			 C *in, C *out,					\
+			 int sign, unsigned flags);			\
+X(plan) X(plan_guru_split_dft)(int rank, const X(iodim) *dims,		\
+			 int howmany_rank,				\
+			 const X(iodim) *howmany_dims,			\
+			 R *ri, R *ii, R *ro, R *io,			\
+			 unsigned flags);				\
+									\
+void X(execute_dft)(const X(plan) p, C *in, C *out);			\
+void X(execute_split_dft)(const X(plan) p, R *ri, R *ii, R *ro, R *io);	\
+									\
+X(plan) X(plan_many_dft_r2c)(int rank, const int *n,			\
+                             int howmany,				\
+                             R *in, const int *inembed,			\
+                             int istride, int idist,			\
+                             C *out, const int *onembed,		\
+                             int ostride, int odist,			\
+                             unsigned flags);				\
+									\
+X(plan) X(plan_dft_r2c)(int rank, const int *n,				\
+                        R *in, C *out, unsigned flags);			\
+									\
+X(plan) X(plan_dft_r2c_1d)(int n,R *in,C *out,unsigned flags);		\
+X(plan) X(plan_dft_r2c_2d)(int nx, int ny,				\
+			   R *in, C *out, unsigned flags);		\
+X(plan) X(plan_dft_r2c_3d)(int nx, int ny,				\
+			   int nz,					\
+			   R *in, C *out, unsigned flags);		\
+									\
+									\
+X(plan) X(plan_many_dft_c2r)(int rank, const int *n,			\
+			     int howmany,				\
+			     C *in, const int *inembed,			\
+			     int istride, int idist,			\
+			     R *out, const int *onembed,		\
+			     int ostride, int odist,			\
+			     unsigned flags);				\
+									\
+X(plan) X(plan_dft_c2r)(int rank, const int *n,				\
+                        C *in, R *out, unsigned flags);			\
+									\
+X(plan) X(plan_dft_c2r_1d)(int n,C *in,R *out,unsigned flags);		\
+X(plan) X(plan_dft_c2r_2d)(int nx, int ny,				\
+			   C *in, R *out, unsigned flags);		\
+X(plan) X(plan_dft_c2r_3d)(int nx, int ny,				\
+			   int nz,					\
+			   C *in, R *out, unsigned flags);		\
+									\
+X(plan) X(plan_guru_dft_r2c)(int rank, const X(iodim) *dims,		\
+			     int howmany_rank,				\
+			     const X(iodim) *howmany_dims,		\
+			     R *in, C *out,				\
+			     unsigned flags);				\
+X(plan) X(plan_guru_dft_c2r)(int rank, const X(iodim) *dims,		\
+			     int howmany_rank,				\
+			     const X(iodim) *howmany_dims,		\
+			     C *in, R *out,				\
+			     unsigned flags);				\
+									\
+X(plan) X(plan_guru_split_dft_r2c)(int rank, const X(iodim) *dims,	\
+			     int howmany_rank,				\
+			     const X(iodim) *howmany_dims,		\
+			     R *in, R *ro, R *io,			\
+			     unsigned flags);				\
+X(plan) X(plan_guru_split_dft_c2r)(int rank, const X(iodim) *dims,	\
+			     int howmany_rank,				\
+			     const X(iodim) *howmany_dims,		\
+			     R *ri, R *ii, R *out,			\
+			     unsigned flags);				\
+									\
+void X(execute_dft_r2c)(const X(plan) p, R *in, C *out);		\
+void X(execute_dft_c2r)(const X(plan) p, C *in, R *out);		\
+									\
+void X(execute_split_dft_r2c)(const X(plan) p, R *in, R *ro, R *io);	\
+void X(execute_split_dft_c2r)(const X(plan) p, R *ri, R *ii, R *out);	\
+									\
+X(plan) X(plan_many_r2r)(int rank, const int *n,			\
+                         int howmany,					\
+                         R *in, const int *inembed,			\
+                         int istride, int idist,			\
+                         R *out, const int *onembed,			\
+                         int ostride, int odist,			\
+                         const X(r2r_kind) *kind, unsigned flags);	\
+									\
+X(plan) X(plan_r2r)(int rank, const int *n, R *in, R *out,		\
+                    const X(r2r_kind) *kind, unsigned flags);		\
+									\
+X(plan) X(plan_r2r_1d)(int n, R *in, R *out,				\
+                       X(r2r_kind) kind, unsigned flags);		\
+X(plan) X(plan_r2r_2d)(int nx, int ny, R *in, R *out,			\
+                       X(r2r_kind) kindx, X(r2r_kind) kindy,		\
+                       unsigned flags);					\
+X(plan) X(plan_r2r_3d)(int nx, int ny, int nz,				\
+                       R *in, R *out, X(r2r_kind) kindx,		\
+                       X(r2r_kind) kindy, X(r2r_kind) kindz,		\
+                       unsigned flags);					\
+									\
+X(plan) X(plan_guru_r2r)(int rank, const X(iodim) *dims,		\
+                         int howmany_rank,				\
+                         const X(iodim) *howmany_dims,			\
+                         R *in, R *out,					\
+                         const X(r2r_kind) *kind, unsigned flags);	\
+void X(execute_r2r)(const X(plan) p, R *in, R *out);			\
+									\
+void X(destroy_plan)(X(plan) p);					\
+void X(forget_wisdom)(void);						\
+void X(cleanup)(void);							\
+									\
+void X(plan_with_nthreads)(int nthreads);				\
+int X(init_threads)(void);						\
+void X(cleanup_threads)(void);						\
+									\
+void X(export_wisdom_to_file)(FILE *output_file);			\
+char *X(export_wisdom_to_string)(void);					\
+void X(export_wisdom)(void (*write_char)(char c, void *), void *data);	\
+int X(import_system_wisdom)(void);					\
+int X(import_wisdom_from_file)(FILE *input_file);			\
+int X(import_wisdom_from_string)(const char *input_string);		\
+int X(import_wisdom)(int (*read_char)(void *), void *data);		\
+									\
+void X(fprint_plan)(const X(plan) p, FILE *output_file);		\
+void X(print_plan)(const X(plan) p);					\
+									\
+void *X(malloc)(size_t n);						\
+void X(free)(void *p);							\
+									\
+void X(flops)(const X(plan) p, double *add, double *mul, double *fma);	\
+									\
+extern const char X(version)[];						\
+extern const char X(cc)[];						\
+extern const char X(codelet_optim)[];
+
+
+/* end of FFTW_DEFINE_API macro */
+
+FFTW_DEFINE_API(FFTW_MANGLE_DOUBLE, double, fftw_complex)
+FFTW_DEFINE_API(FFTW_MANGLE_FLOAT, float, fftwf_complex)
+FFTW_DEFINE_API(FFTW_MANGLE_LONG_DOUBLE, long double, fftwl_complex)
+
+#define FFTW_FORWARD (-1)
+#define FFTW_BACKWARD (+1)
+
+/* documented flags */
+#define FFTW_MEASURE (0U)
+#define FFTW_DESTROY_INPUT (1U << 0)
+#define FFTW_UNALIGNED (1U << 1)
+#define FFTW_CONSERVE_MEMORY (1U << 2)
+#define FFTW_EXHAUSTIVE (1U << 3) /* NO_EXHAUSTIVE is default */
+#define FFTW_PRESERVE_INPUT (1U << 4) /* cancels FFTW_DESTROY_INPUT */
+#define FFTW_PATIENT (1U << 5) /* IMPATIENT is default */
+#define FFTW_ESTIMATE (1U << 6)
+
+/* undocumented beyond-guru flags */
+#define FFTW_ESTIMATE_PATIENT (1U << 7)
+#define FFTW_BELIEVE_PCOST (1U << 8)
+#define FFTW_DFT_R2HC_ICKY (1U << 9)
+#define FFTW_NONTHREADED_ICKY (1U << 10)
+#define FFTW_NO_BUFFERING (1U << 11)
+#define FFTW_NO_INDIRECT_OP (1U << 12)
+#define FFTW_ALLOW_LARGE_GENERIC (1U << 13) /* NO_LARGE_GENERIC is default */
+#define FFTW_NO_RANK_SPLITS (1U << 14)
+#define FFTW_NO_VRANK_SPLITS (1U << 15)
+#define FFTW_NO_VRECURSE (1U << 16)
+
+#define FFTW_NO_SIMD (1U << 17)
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif /* __cplusplus */
+
+#endif /* FFTW3_H */
diff --git a/src/fftw3/api/flops.c b/src/fftw3/api/flops.c
new file mode 100644
index 0000000..43e8891
--- /dev/null
+++ b/src/fftw3/api/flops.c
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+void X(flops)(const X(plan) p, double *add, double *mul, double *fma)
+{
+     opcnt *o = &p->pln->ops;
+     *add = o->add; *mul = o->mul; *fma = o->fma;
+}
diff --git a/src/fftw3/api/forget-wisdom.c b/src/fftw3/api/forget-wisdom.c
new file mode 100644
index 0000000..10ea580
--- /dev/null
+++ b/src/fftw3/api/forget-wisdom.c
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+void X(forget_wisdom)(void)
+{
+     planner *plnr = X(the_planner)();
+     plnr->adt->forget(plnr, FORGET_EVERYTHING);
+}
diff --git a/src/fftw3/api/import-system-wisdom.c b/src/fftw3/api/import-system-wisdom.c
new file mode 100644
index 0000000..393b3a6
--- /dev/null
+++ b/src/fftw3/api/import-system-wisdom.c
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+#if defined(FFTW_SINGLE)
+#  define WISDOM_NAME "wisdomf"
+#elif defined(FFTW_LDOUBLE)
+#  define WISDOM_NAME "wisdoml"
+#else
+#  define WISDOM_NAME "wisdom"
+#endif
+
+int X(import_system_wisdom)(void)
+{
+#if defined(__WIN32__) || defined(WIN32) || defined(_WINDOWS)
+     return 0; /* TODO? */
+#else
+
+     FILE *f;
+     f = fopen("/etc/fftw/" WISDOM_NAME, "r");
+     if (f) {
+          int ret = X(import_wisdom_from_file)(f);
+          fclose(f);
+          return ret;
+     } else
+          return 0;
+#endif
+}
diff --git a/src/fftw3/api/import-wisdom-from-file.c b/src/fftw3/api/import-wisdom-from-file.c
new file mode 100644
index 0000000..e682ab9
--- /dev/null
+++ b/src/fftw3/api/import-wisdom-from-file.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include <stdio.h>
+
+/* getc()/putc() are *unbelievably* slow on linux.  Looks like glibc
+   is grabbing a lock for each call to getc()/putc(), or something
+   like that.  You pay the price for these idiotic posix threads
+   whether you use them or not.
+
+   So, we do our own buffering.  This completely defeats the purpose
+   of having stdio in the first place, of course.
+*/
+  
+#define BUFSZ 256
+
+typedef struct {
+     scanner super;
+     FILE *f;
+     char buf[BUFSZ];
+     char *bufr, *bufw;
+} S;
+
+static int getchr_file(scanner * sc_)
+{
+     S *sc = (S *) sc_;
+
+     if (sc->bufr >= sc->bufw) {
+	  sc->bufr = sc->buf;
+	  sc->bufw = sc->buf + fread(sc->buf, 1, BUFSZ, sc->f);
+	  if (sc->bufr >= sc->bufw)
+	       return EOF;
+     }
+
+     return *(sc->bufr++);
+}
+
+static scanner *mkscanner_file(FILE *f)
+{
+     S *sc = (S *) X(mkscanner)(sizeof(S), getchr_file);
+     sc->f = f;
+     sc->bufr = sc->bufw = sc->buf;
+     return &sc->super;
+}
+
+int X(import_wisdom_from_file)(FILE *input_file)
+{
+     scanner *s = mkscanner_file(input_file);
+     planner *plnr = X(the_planner)();
+     int ret = plnr->adt->imprt(plnr, s);
+     X(scanner_destroy)(s);
+     return ret;
+}
diff --git a/src/fftw3/api/import-wisdom-from-string.c b/src/fftw3/api/import-wisdom-from-string.c
new file mode 100644
index 0000000..d1ca6b8
--- /dev/null
+++ b/src/fftw3/api/import-wisdom-from-string.c
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+typedef struct {
+     scanner super;
+     const char *s;
+} S_str;
+
+static int getchr_str(scanner * sc_)
+{
+     S_str *sc = (S_str *) sc_;
+     if (!*sc->s)
+          return EOF;
+     return *sc->s++;
+}
+
+static scanner *mkscanner_str(const char *s)
+{
+     S_str *sc = (S_str *) X(mkscanner)(sizeof(S_str), getchr_str);
+     sc->s = s;
+     return &sc->super;
+}
+
+int X(import_wisdom_from_string)(const char *input_string)
+{
+     scanner *s = mkscanner_str(input_string);
+     planner *plnr = X(the_planner)();
+     int ret = plnr->adt->imprt(plnr, s);
+     X(scanner_destroy)(s);
+     return ret;
+}
diff --git a/src/fftw3/api/import-wisdom.c b/src/fftw3/api/import-wisdom.c
new file mode 100644
index 0000000..505ebab
--- /dev/null
+++ b/src/fftw3/api/import-wisdom.c
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+typedef struct {
+     scanner super;
+     int (*read_char)(void *);
+     void *data;
+} S;
+
+static int getchr_generic(scanner * s_)
+{
+     S *s = (S *) s_;
+     return (s->read_char)(s->data);
+}
+
+int X(import_wisdom)(int (*read_char)(void *), void *data)
+{
+     S *s = (S *) X(mkscanner)(sizeof(S), getchr_generic);
+     planner *plnr = X(the_planner)();
+     int ret;
+
+     s->read_char = read_char;
+     s->data = data;
+     ret = plnr->adt->imprt(plnr, (scanner *) s);
+     X(scanner_destroy)((scanner *) s);
+     return ret;
+}
diff --git a/src/fftw3/api/map-r2r-kind.c b/src/fftw3/api/map-r2r-kind.c
new file mode 100644
index 0000000..adba2ce
--- /dev/null
+++ b/src/fftw3/api/map-r2r-kind.c
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+rdft_kind *X(map_r2r_kind)(int rank, const X(r2r_kind) * kind)
+{
+     int i;
+     rdft_kind *k;
+
+     A(FINITE_RNK(rank));
+     k = (rdft_kind *) MALLOC(rank * sizeof(rdft_kind), PROBLEMS);
+     for (i = 0; i < rank; ++i) {
+	  rdft_kind m;
+          switch (kind[i]) {
+	      case FFTW_R2HC: m = R2HC; break;
+	      case FFTW_HC2R: m = HC2R; break;
+	      case FFTW_DHT: m = DHT; break;
+	      case FFTW_REDFT00: m = REDFT00; break;
+	      case FFTW_REDFT01: m = REDFT01; break;
+	      case FFTW_REDFT10: m = REDFT10; break;
+	      case FFTW_REDFT11: m = REDFT11; break;
+	      case FFTW_RODFT00: m = RODFT00; break;
+	      case FFTW_RODFT01: m = RODFT01; break;
+	      case FFTW_RODFT10: m = RODFT10; break;
+	      case FFTW_RODFT11: m = RODFT11; break;
+	      default: m = R2HC; A(0);
+          }
+	  k[i] = m;
+     }
+     return k;
+}
diff --git a/src/fftw3/api/mapflags.c b/src/fftw3/api/mapflags.c
new file mode 100644
index 0000000..abb4e6b
--- /dev/null
+++ b/src/fftw3/api/mapflags.c
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+/* a flag operation: x is either a flag, in which case xm == 0, or
+   a mask, in which case xm == x; using this we can compactly code
+   the various bit operations via (flags & x) ^ xm or (flags | x) ^ xm. */
+typedef struct {
+     unsigned x, xm;
+} flagmask;
+
+typedef struct {
+     flagmask flag;
+     flagmask op;
+} flagop;
+
+#define FLAGP(f, msk)(((f) & (msk).x) ^ (msk).xm)
+#define OP(f, msk)(((f) | (msk).x) ^ (msk).xm)
+
+#define YES(x) {x, 0}
+#define NO(x) {x, x}
+#define IMPLIES(predicate, consequence) { predicate, consequence }
+#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b))
+#define NEQV(a, b) IMPLIES(YES(a), NO(b)), IMPLIES(NO(a), YES(b))
+
+static void map_flags(unsigned *iflags, unsigned *oflags,
+		      const flagop flagmap[], int nmap)
+{
+     int i;
+     for (i = 0; i < nmap; ++i)
+          if (FLAGP(*iflags, flagmap[i].flag))
+               *oflags = OP(*oflags, flagmap[i].op);
+}
+
+#define NELEM(array)(sizeof(array) / sizeof((array)[0]))
+
+void X(mapflags)(planner *plnr, unsigned flags)
+{
+     unsigned tmpflags;
+
+     /* map of api flags -> api flags, to implement consistency rules
+        and combination flags */
+     const flagop self_flagmap[] = {
+	  /* in some cases (notably for halfcomplex->real transforms),
+	     DESTROY_INPUT is the default, so we need to support
+	     an inverse flag to disable it: */
+	  IMPLIES(YES(FFTW_PRESERVE_INPUT), NO(FFTW_DESTROY_INPUT)),
+
+	  IMPLIES(YES(FFTW_EXHAUSTIVE), YES(FFTW_PATIENT)),
+
+	  IMPLIES(YES(FFTW_ESTIMATE), NO(FFTW_PATIENT)),
+	  IMPLIES(YES(FFTW_ESTIMATE),
+		  YES(FFTW_ESTIMATE_PATIENT | FFTW_NO_INDIRECT_OP)),
+
+	  /* a canonical set of fftw2-like impatience flags */
+	  IMPLIES(NO(FFTW_PATIENT),
+		  YES(FFTW_NO_VRECURSE
+		      | FFTW_NO_RANK_SPLITS
+		      | FFTW_NO_VRANK_SPLITS
+		      | FFTW_NONTHREADED_ICKY
+		      | FFTW_DFT_R2HC_ICKY
+		      | FFTW_BELIEVE_PCOST))
+     };
+
+     /* map of (processed) api flags to internal problem/planner flags */
+     const flagop problem_flagmap[] = {
+	  EQV(FFTW_DESTROY_INPUT, DESTROY_INPUT),
+	  EQV(FFTW_NO_SIMD, NO_SIMD),
+	  EQV(FFTW_CONSERVE_MEMORY, CONSERVE_MEMORY)
+     };
+     const flagop planner_flagmap[] = {
+	  NEQV(FFTW_EXHAUSTIVE, NO_EXHAUSTIVE),
+
+	  /* the following are undocumented, "beyond-guru" flags that
+	     require some understanding of FFTW internals */
+	  EQV(FFTW_ESTIMATE_PATIENT, ESTIMATE),
+	  EQV(FFTW_BELIEVE_PCOST, BELIEVE_PCOST),
+	  EQV(FFTW_DFT_R2HC_ICKY, DFT_R2HC_ICKY),
+	  EQV(FFTW_NONTHREADED_ICKY, NONTHREADED_ICKY),
+	  EQV(FFTW_NO_BUFFERING, NO_BUFFERING),
+	  EQV(FFTW_NO_INDIRECT_OP, NO_INDIRECT_OP),
+	  NEQV(FFTW_ALLOW_LARGE_GENERIC, NO_LARGE_GENERIC),
+	  EQV(FFTW_NO_RANK_SPLITS, NO_RANK_SPLITS),
+	  EQV(FFTW_NO_VRANK_SPLITS, NO_VRANK_SPLITS),
+	  EQV(FFTW_NO_VRECURSE, NO_VRECURSE)
+     };
+
+     map_flags(&flags, &flags, self_flagmap, NELEM(self_flagmap));
+
+     tmpflags = 0;
+     map_flags(&flags, &tmpflags, problem_flagmap, NELEM(problem_flagmap));
+     plnr->problem_flags = tmpflags;
+
+     tmpflags = 0;
+     map_flags(&flags, &tmpflags, planner_flagmap, NELEM(planner_flagmap));
+     plnr->planner_flags = tmpflags;
+}
diff --git a/src/fftw3/api/mkprinter-file.c b/src/fftw3/api/mkprinter-file.c
new file mode 100644
index 0000000..24476f9
--- /dev/null
+++ b/src/fftw3/api/mkprinter-file.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include <stdio.h>
+
+#define BUFSZ 256
+
+typedef struct {
+     printer super;
+     FILE *f;
+     char buf[BUFSZ];
+     char *bufw;
+} P;
+
+static void myflush(P *p)
+{
+     fwrite(p->buf, 1, p->bufw - p->buf, p->f);
+     p->bufw = p->buf;
+}
+
+static void myputchr(printer *p_, char c)
+{
+     P *p = (P *) p_;
+     if (p->bufw >= p->buf + BUFSZ)
+	  myflush(p);
+     *p->bufw++ = c;
+}
+
+static void mycleanup(printer *p_)
+{
+     P *p = (P *) p_;
+     myflush(p);
+}
+
+printer *X(mkprinter_file)(FILE *f)
+{
+     P *p = (P *) X(mkprinter)(sizeof(P), myputchr, mycleanup);
+     p->f = f;
+     p->bufw = p->buf;
+     return &p->super;
+}
diff --git a/src/fftw3/api/mktensor-iodims.c b/src/fftw3/api/mktensor-iodims.c
new file mode 100644
index 0000000..99ce49e
--- /dev/null
+++ b/src/fftw3/api/mktensor-iodims.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+tensor *X(mktensor_iodims)(int rank, const X(iodim) *dims, int is, int os)
+{
+     int i;
+     tensor *x = X(mktensor)(rank);
+
+     if (FINITE_RNK(rank)) {
+          for (i = 0; i < rank; ++i) {
+               x->dims[i].n = dims[i].n;
+               x->dims[i].is = dims[i].is * is;
+               x->dims[i].os = dims[i].os * os;
+          }
+     }
+     return x;
+}
+
+static int iodims_kosherp(int rank, const X(iodim) *dims, int allow_minfty)
+{
+     int i;
+
+     if (rank < 0) return 0;
+
+     if (allow_minfty) {
+	  if (!FINITE_RNK(rank)) return 1;
+	  for (i = 0; i < rank; ++i)
+	       if (dims[i].n < 0) return 0;
+     } else {
+	  if (!FINITE_RNK(rank)) return 0;
+	  for (i = 0; i < rank; ++i)
+	       if (dims[i].n <= 0) return 0;
+     }
+
+     return 1;
+}
+
+int X(guru_kosherp)(int rank, const X(iodim) *dims,
+		    int howmany_rank, const X(iodim) *howmany_dims)
+{
+     return (iodims_kosherp(rank, dims, 0) &&
+	     iodims_kosherp(howmany_rank, howmany_dims, 1));
+}
diff --git a/src/fftw3/api/mktensor-rowmajor.c b/src/fftw3/api/mktensor-rowmajor.c
new file mode 100644
index 0000000..84b87ca
--- /dev/null
+++ b/src/fftw3/api/mktensor-rowmajor.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+tensor *X(mktensor_rowmajor)(int rnk, const int *n,
+			     const int *niphys, const int *nophys,
+			     int is, int os)
+{
+     tensor *x = X(mktensor)(rnk);
+
+     if (FINITE_RNK(rnk) && rnk > 0) {
+          int i;
+
+          A(n && niphys && nophys);
+          x->dims[rnk - 1].is = is;
+          x->dims[rnk - 1].os = os;
+          x->dims[rnk - 1].n = n[rnk - 1];
+          for (i = rnk - 1; i > 0; --i) {
+               x->dims[i - 1].is = x->dims[i].is * niphys[i];
+               x->dims[i - 1].os = x->dims[i].os * nophys[i];
+               x->dims[i - 1].n = n[i - 1];
+          }
+     }
+     return x;
+}
+
+static int rowmajor_kosherp(int rnk, const int *n)
+{
+     int i;
+
+     if (!FINITE_RNK(rnk)) return 0;
+     if (rnk < 0) return 0;
+
+     for (i = 0; i < rnk; ++i)
+	  if (n[i] <= 0) return 0;
+
+     return 1;
+}
+
+int X(many_kosherp)(int rnk, const int *n, int howmany)
+{
+     return (howmany >= 0) && rowmajor_kosherp(rnk, n);
+}
diff --git a/src/fftw3/api/plan-dft-1d.c b/src/fftw3/api/plan-dft-1d.c
new file mode 100644
index 0000000..02ff69d
--- /dev/null
+++ b/src/fftw3/api/plan-dft-1d.c
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "dft.h"
+
+X(plan) X(plan_dft_1d)(int n, C *in, C *out, int sign, unsigned flags)
+{
+     return X(plan_dft)(1, &n, in, out, sign, flags);
+}
diff --git a/src/fftw3/api/plan-dft-2d.c b/src/fftw3/api/plan-dft-2d.c
new file mode 100644
index 0000000..f41e7e3
--- /dev/null
+++ b/src/fftw3/api/plan-dft-2d.c
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "dft.h"
+
+X(plan) X(plan_dft_2d)(int nx, int ny, C *in, C *out, int sign, unsigned flags)
+{
+     int n[2];
+     n[0] = nx;
+     n[1] = ny;
+     return X(plan_dft)(2, n, in, out, sign, flags);
+}
diff --git a/src/fftw3/api/plan-dft-3d.c b/src/fftw3/api/plan-dft-3d.c
new file mode 100644
index 0000000..a605a46
--- /dev/null
+++ b/src/fftw3/api/plan-dft-3d.c
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "dft.h"
+
+X(plan) X(plan_dft_3d)(int nx, int ny, int nz,
+		       C *in, C *out, int sign, unsigned flags)
+{
+     int n[3];
+     n[0] = nx;
+     n[1] = ny;
+     n[2] = nz;
+     return X(plan_dft)(3, n, in, out, sign, flags);
+}
diff --git a/src/fftw3/api/plan-dft-c2r-1d.c b/src/fftw3/api/plan-dft-c2r-1d.c
new file mode 100644
index 0000000..7dc9a93
--- /dev/null
+++ b/src/fftw3/api/plan-dft-c2r-1d.c
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_dft_c2r_1d)(int n, C *in, R *out, unsigned flags)
+{
+     return X(plan_dft_c2r)(1, &n, in, out, flags);
+}
diff --git a/src/fftw3/api/plan-dft-c2r-2d.c b/src/fftw3/api/plan-dft-c2r-2d.c
new file mode 100644
index 0000000..37d39bb
--- /dev/null
+++ b/src/fftw3/api/plan-dft-c2r-2d.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_dft_c2r_2d)(int nx, int ny, C *in, R *out, unsigned flags)
+{
+     int n[2];
+     n[0] = nx;
+     n[1] = ny;
+     return X(plan_dft_c2r)(2, n, in, out, flags);
+}
diff --git a/src/fftw3/api/plan-dft-c2r-3d.c b/src/fftw3/api/plan-dft-c2r-3d.c
new file mode 100644
index 0000000..4644126
--- /dev/null
+++ b/src/fftw3/api/plan-dft-c2r-3d.c
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_dft_c2r_3d)(int nx, int ny, int nz,
+			   C *in, R *out, unsigned flags)
+{
+     int n[3];
+     n[0] = nx;
+     n[1] = ny;
+     n[2] = nz;
+     return X(plan_dft_c2r)(3, n, in, out, flags);
+}
diff --git a/src/fftw3/api/plan-dft-c2r.c b/src/fftw3/api/plan-dft-c2r.c
new file mode 100644
index 0000000..118c1cb
--- /dev/null
+++ b/src/fftw3/api/plan-dft-c2r.c
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_dft_c2r)(int rank, const int *n, C *in, R *out, unsigned flags)
+{
+     return X(plan_many_dft_c2r)(rank, n, 1,
+				 in, 0, 1, 1, out, 0, 1, 1, flags);
+}
diff --git a/src/fftw3/api/plan-dft-r2c-1d.c b/src/fftw3/api/plan-dft-r2c-1d.c
new file mode 100644
index 0000000..5d4865a
--- /dev/null
+++ b/src/fftw3/api/plan-dft-r2c-1d.c
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_dft_r2c_1d)(int n, R *in, C *out, unsigned flags)
+{
+     return X(plan_dft_r2c)(1, &n, in, out, flags);
+}
diff --git a/src/fftw3/api/plan-dft-r2c-2d.c b/src/fftw3/api/plan-dft-r2c-2d.c
new file mode 100644
index 0000000..289841c
--- /dev/null
+++ b/src/fftw3/api/plan-dft-r2c-2d.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_dft_r2c_2d)(int nx, int ny, R *in, C *out, unsigned flags)
+{
+     int n[2];
+     n[0] = nx;
+     n[1] = ny;
+     return X(plan_dft_r2c)(2, n, in, out, flags);
+}
diff --git a/src/fftw3/api/plan-dft-r2c-3d.c b/src/fftw3/api/plan-dft-r2c-3d.c
new file mode 100644
index 0000000..c25d04f
--- /dev/null
+++ b/src/fftw3/api/plan-dft-r2c-3d.c
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_dft_r2c_3d)(int nx, int ny, int nz,
+			   R *in, C *out, unsigned flags)
+{
+     int n[3];
+     n[0] = nx;
+     n[1] = ny;
+     n[2] = nz;
+     return X(plan_dft_r2c)(3, n, in, out, flags);
+}
diff --git a/src/fftw3/api/plan-dft-r2c.c b/src/fftw3/api/plan-dft-r2c.c
new file mode 100644
index 0000000..8624995
--- /dev/null
+++ b/src/fftw3/api/plan-dft-r2c.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_dft_r2c)(int rank, const int *n, R *in, C *out, unsigned flags)
+{
+     return X(plan_many_dft_r2c)(rank, n, 1,
+				 in, 0, 1, 1, 
+				 out, 0, 1, 1, 
+				 flags);
+}
diff --git a/src/fftw3/api/plan-dft.c b/src/fftw3/api/plan-dft.c
new file mode 100644
index 0000000..f9c8756
--- /dev/null
+++ b/src/fftw3/api/plan-dft.c
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_dft)(int rank, const int *n,
+		    C *in, C *out, int sign, unsigned flags)
+{
+     return X(plan_many_dft)(rank, n, 1,
+			     in, 0, 1, 1, 
+			     out, 0, 1, 1, 
+			     sign, flags);
+}
diff --git a/src/fftw3/api/plan-guru-dft-c2r.c b/src/fftw3/api/plan-guru-dft-c2r.c
new file mode 100644
index 0000000..0ab3a9c
--- /dev/null
+++ b/src/fftw3/api/plan-guru-dft-c2r.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+X(plan) X(plan_guru_dft_c2r)(int rank, const X(iodim) *dims,
+			     int howmany_rank, const X(iodim) *howmany_dims,
+			     C *in, R *out, unsigned flags)
+{
+     R *ri, *ii;
+
+     if (!X(guru_kosherp)(rank, dims, howmany_rank, howmany_dims)) return 0;
+
+     X(extract_reim)(FFT_SIGN, in, &ri, &ii);
+
+     if (out != ri)
+	  flags |= FFTW_DESTROY_INPUT;
+     return X(mkapiplan)(
+	  0, flags, 
+	  X(mkproblem_rdft2_d)(X(mktensor_iodims)(rank, dims, 2, 1),
+			       X(mktensor_iodims)(howmany_rank, howmany_dims,
+						  2, 1),
+			       TAINT_UNALIGNED(out, flags),
+			       TAINT_UNALIGNED(ri, flags),
+			       TAINT_UNALIGNED(ii, flags), HC2R));
+}
diff --git a/src/fftw3/api/plan-guru-dft-r2c.c b/src/fftw3/api/plan-guru-dft-r2c.c
new file mode 100644
index 0000000..c4e4d41
--- /dev/null
+++ b/src/fftw3/api/plan-guru-dft-r2c.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+X(plan) X(plan_guru_dft_r2c)(int rank, const X(iodim) *dims,
+			     int howmany_rank,
+			     const X(iodim) *howmany_dims,
+			     R *in, C *out, unsigned flags)
+{
+     R *ro, *io;
+
+     if (!X(guru_kosherp)(rank, dims, howmany_rank, howmany_dims)) return 0;
+
+     X(extract_reim)(FFT_SIGN, out, &ro, &io);
+
+     return X(mkapiplan)(
+	  0, flags,
+	  X(mkproblem_rdft2_d)(X(mktensor_iodims)(rank, dims, 1, 2),
+			       X(mktensor_iodims)(howmany_rank, howmany_dims,
+						  1, 2),
+			       TAINT_UNALIGNED(in, flags),
+			       TAINT_UNALIGNED(ro, flags),
+			       TAINT_UNALIGNED(io, flags), R2HC));
+}
diff --git a/src/fftw3/api/plan-guru-dft.c b/src/fftw3/api/plan-guru-dft.c
new file mode 100644
index 0000000..32c894b
--- /dev/null
+++ b/src/fftw3/api/plan-guru-dft.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "dft.h"
+
+X(plan) X(plan_guru_dft)(int rank, const X(iodim) *dims,
+			 int howmany_rank, const X(iodim) *howmany_dims,
+			 C *in, C *out, int sign, unsigned flags)
+{
+     R *ri, *ii, *ro, *io;
+
+     if (!X(guru_kosherp)(rank, dims, howmany_rank, howmany_dims)) return 0;
+
+     X(extract_reim)(sign, in, &ri, &ii);
+     X(extract_reim)(sign, out, &ro, &io);
+
+     return X(mkapiplan)(
+	  sign, flags,
+	  X(mkproblem_dft_d)(X(mktensor_iodims)(rank, dims, 2, 2),
+			     X(mktensor_iodims)(howmany_rank, howmany_dims,
+						2, 2),
+			     TAINT_UNALIGNED(ri, flags),
+			     TAINT_UNALIGNED(ii, flags), 
+			     TAINT_UNALIGNED(ro, flags),
+			     TAINT_UNALIGNED(io, flags)));
+}
diff --git a/src/fftw3/api/plan-guru-r2r.c b/src/fftw3/api/plan-guru-r2r.c
new file mode 100644
index 0000000..d5f2120
--- /dev/null
+++ b/src/fftw3/api/plan-guru-r2r.c
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+rdft_kind *X(map_r2r_kind)(int rank, const X(r2r_kind) * kind);
+
+X(plan) X(plan_guru_r2r)(int rank, const X(iodim) *dims,
+			 int howmany_rank,
+			 const X(iodim) *howmany_dims,
+			 R *in, R *out,
+			 const X(r2r_kind) * kind, unsigned flags)
+{
+     X(plan) p;
+     rdft_kind *k;
+
+     if (!X(guru_kosherp)(rank, dims, howmany_rank, howmany_dims)) return 0;
+
+     k = X(map_r2r_kind)(rank, kind);
+     p = X(mkapiplan)(
+	  0, flags,
+	  X(mkproblem_rdft_d)(X(mktensor_iodims)(rank, dims, 1, 1),
+			      X(mktensor_iodims)(howmany_rank, howmany_dims,
+						 1, 1), 
+			      TAINT_UNALIGNED(in, flags),
+			      TAINT_UNALIGNED(out, flags), k));
+     X(ifree0)(k);
+     return p;
+}
diff --git a/src/fftw3/api/plan-guru-split-dft-c2r.c b/src/fftw3/api/plan-guru-split-dft-c2r.c
new file mode 100644
index 0000000..f51bc9a
--- /dev/null
+++ b/src/fftw3/api/plan-guru-split-dft-c2r.c
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+X(plan) X(plan_guru_split_dft_c2r)(int rank, const X(iodim) *dims,
+			     int howmany_rank, const X(iodim) *howmany_dims,
+			     R *ri, R *ii, R *out, unsigned flags)
+{
+     if (!X(guru_kosherp)(rank, dims, howmany_rank, howmany_dims)) return 0;
+
+     if (out != ri)
+	  flags |= FFTW_DESTROY_INPUT;
+     return X(mkapiplan)(
+	  0, flags, 
+	  X(mkproblem_rdft2_d)(X(mktensor_iodims)(rank, dims, 1, 1),
+			       X(mktensor_iodims)(howmany_rank, howmany_dims,
+						  1, 1),
+			       TAINT_UNALIGNED(out, flags),
+			       TAINT_UNALIGNED(ri, flags),
+			       TAINT_UNALIGNED(ii, flags), HC2R));
+}
diff --git a/src/fftw3/api/plan-guru-split-dft-r2c.c b/src/fftw3/api/plan-guru-split-dft-r2c.c
new file mode 100644
index 0000000..3c5eafd
--- /dev/null
+++ b/src/fftw3/api/plan-guru-split-dft-r2c.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+X(plan) X(plan_guru_split_dft_r2c)(int rank, const X(iodim) *dims,
+				   int howmany_rank,
+				   const X(iodim) *howmany_dims,
+				   R *in, R *ro, R *io, unsigned flags)
+{
+     if (!X(guru_kosherp)(rank, dims, howmany_rank, howmany_dims)) return 0;
+
+     return X(mkapiplan)(
+	  0, flags,
+	  X(mkproblem_rdft2_d)(X(mktensor_iodims)(rank, dims, 1, 1),
+			       X(mktensor_iodims)(howmany_rank, howmany_dims,
+						  1, 1),
+			       TAINT_UNALIGNED(in, flags),
+			       TAINT_UNALIGNED(ro, flags),
+			       TAINT_UNALIGNED(io, flags), R2HC));
+}
diff --git a/src/fftw3/api/plan-guru-split-dft.c b/src/fftw3/api/plan-guru-split-dft.c
new file mode 100644
index 0000000..62437c2
--- /dev/null
+++ b/src/fftw3/api/plan-guru-split-dft.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "dft.h"
+
+X(plan) X(plan_guru_split_dft)(int rank, const X(iodim) *dims,
+			       int howmany_rank, const X(iodim) *howmany_dims,
+			       R *ri, R *ii, R *ro, R *io, unsigned flags)
+{
+     if (!X(guru_kosherp)(rank, dims, howmany_rank, howmany_dims)) return 0;
+
+     return X(mkapiplan)(
+	  ii - ri == 1 && io - ro == 1 ? FFT_SIGN : -FFT_SIGN, flags,
+	  X(mkproblem_dft_d)(X(mktensor_iodims)(rank, dims, 1, 1),
+			     X(mktensor_iodims)(howmany_rank, howmany_dims,
+						1, 1),
+			     TAINT_UNALIGNED(ri, flags),
+			     TAINT_UNALIGNED(ii, flags), 
+			     TAINT_UNALIGNED(ro, flags),
+			     TAINT_UNALIGNED(io, flags)));
+}
diff --git a/src/fftw3/api/plan-many-dft-c2r.c b/src/fftw3/api/plan-many-dft-c2r.c
new file mode 100644
index 0000000..81b6338
--- /dev/null
+++ b/src/fftw3/api/plan-many-dft-c2r.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+X(plan) X(plan_many_dft_c2r)(int rank, const int *n,
+			     int howmany,
+			     C *in, const int *inembed,
+			     int istride, int idist,
+			     R *out, const int *onembed,
+			     int ostride, int odist, unsigned flags)
+{
+     R *ri, *ii;
+     int *nfi, *nfo;
+     int inplace;
+     X(plan) p;
+
+     if (!X(many_kosherp)(rank, n, howmany)) return 0;
+
+     X(extract_reim)(FFT_SIGN, in, &ri, &ii);
+     inplace = out == ri;
+
+     if (!inplace)
+	  flags |= FFTW_DESTROY_INPUT;
+     p = X(mkapiplan)(
+	  0, flags,
+	  X(mkproblem_rdft2_d)(
+	       X(mktensor_rowmajor)(
+		    rank, n, 
+		    X (rdft2_pad)(rank, n, inembed, inplace, 1, &nfi),
+		    X (rdft2_pad)(rank, n, onembed, inplace, 0, &nfo),
+		    2 * istride, ostride),
+	       X(mktensor_1d)(howmany, 2 * idist, odist),
+	       TAINT_UNALIGNED(out, flags),
+	       TAINT_UNALIGNED(ri, flags), TAINT_UNALIGNED(ii, flags),
+	       HC2R));
+
+     X(ifree0)(nfi);
+     X(ifree0)(nfo);
+     return p;
+}
diff --git a/src/fftw3/api/plan-many-dft-r2c.c b/src/fftw3/api/plan-many-dft-r2c.c
new file mode 100644
index 0000000..093d9ba
--- /dev/null
+++ b/src/fftw3/api/plan-many-dft-r2c.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+X(plan) X(plan_many_dft_r2c)(int rank, const int *n,
+			     int howmany,
+			     R *in, const int *inembed,
+			     int istride, int idist,
+			     C *out, const int *onembed,
+			     int ostride, int odist, unsigned flags)
+{
+     R *ro, *io;
+     int *nfi, *nfo;
+     int inplace;
+     X(plan) p;
+
+     if (!X(many_kosherp)(rank, n, howmany)) return 0;
+
+     X(extract_reim)(FFT_SIGN, out, &ro, &io);
+     inplace = in == ro;
+
+     p = X(mkapiplan)(
+	  0, flags, 
+	  X(mkproblem_rdft2_d)(
+	       X(mktensor_rowmajor)(
+		    rank, n,
+		    X(rdft2_pad)(rank, n, inembed, inplace, 0, &nfi),
+		    X(rdft2_pad)(rank, n, onembed, inplace, 1, &nfo),
+		    istride, 2 * ostride), 
+	       X(mktensor_1d)(howmany, idist, 2 * odist),
+	       TAINT_UNALIGNED(in, flags),
+	       TAINT_UNALIGNED(ro, flags), TAINT_UNALIGNED(io, flags),
+	       R2HC));
+
+     X(ifree0)(nfi);
+     X(ifree0)(nfo);
+     return p;
+}
diff --git a/src/fftw3/api/plan-many-dft.c b/src/fftw3/api/plan-many-dft.c
new file mode 100644
index 0000000..56f0ed4
--- /dev/null
+++ b/src/fftw3/api/plan-many-dft.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "dft.h"
+
+#define N0(nembed)((nembed) ? (nembed) : n)
+
+X(plan) X(plan_many_dft)(int rank, const int *n,
+			 int howmany,
+			 C *in, const int *inembed,
+			 int istride, int idist,
+			 C *out, const int *onembed,
+			 int ostride, int odist, int sign, unsigned flags)
+{
+     R *ri, *ii, *ro, *io;
+
+     if (!X(many_kosherp)(rank, n, howmany)) return 0;
+
+     X(extract_reim)(sign, in, &ri, &ii);
+     X(extract_reim)(sign, out, &ro, &io);
+
+     return 
+	  X(mkapiplan)(sign, flags,
+		       X(mkproblem_dft_d)(
+			    X(mktensor_rowmajor)(rank, n, 
+						 N0(inembed), N0(onembed),
+						 2 * istride, 2 * ostride),
+			    X(mktensor_1d)(howmany, 2 * idist, 2 * odist),
+			    TAINT_UNALIGNED(ri, flags),
+			    TAINT_UNALIGNED(ii, flags),
+			    TAINT_UNALIGNED(ro, flags),
+			    TAINT_UNALIGNED(io, flags)));
+}
diff --git a/src/fftw3/api/plan-many-r2r.c b/src/fftw3/api/plan-many-r2r.c
new file mode 100644
index 0000000..0a6815f
--- /dev/null
+++ b/src/fftw3/api/plan-many-r2r.c
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+#include "rdft.h"
+
+rdft_kind *X(map_r2r_kind)(int rank, const X(r2r_kind) * kind);
+
+#define N0(nembed)((nembed) ? (nembed) : n)
+
+X(plan) X(plan_many_r2r)(int rank, const int *n,
+			 int howmany,
+			 R *in, const int *inembed,
+			 int istride, int idist,
+			 R *out, const int *onembed,
+			 int ostride, int odist,
+			 const X(r2r_kind) * kind, unsigned flags)
+{
+     X(plan) p;
+     rdft_kind *k;
+
+     if (!X(many_kosherp)(rank, n, howmany)) return 0;
+
+     k = X(map_r2r_kind)(rank, kind);
+     p = X(mkapiplan)(
+	  0, flags,
+	  X(mkproblem_rdft_d)(X(mktensor_rowmajor)(rank, n, 
+						   N0(inembed), N0(onembed),
+						   istride, ostride),
+			      X(mktensor_1d)(howmany, idist, odist),
+			      TAINT_UNALIGNED(in, flags), 
+			      TAINT_UNALIGNED(out, flags), k));
+     X(ifree0)(k);
+     return p;
+}
diff --git a/src/fftw3/api/plan-r2r-1d.c b/src/fftw3/api/plan-r2r-1d.c
new file mode 100644
index 0000000..372d1c0
--- /dev/null
+++ b/src/fftw3/api/plan-r2r-1d.c
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_r2r_1d)(int n, R *in, R *out, X(r2r_kind) kind, unsigned flags)
+{
+     return X(plan_r2r)(1, &n, in, out, &kind, flags);
+}
diff --git a/src/fftw3/api/plan-r2r-2d.c b/src/fftw3/api/plan-r2r-2d.c
new file mode 100644
index 0000000..7702560
--- /dev/null
+++ b/src/fftw3/api/plan-r2r-2d.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_r2r_2d)(int nx, int ny, R *in, R *out,
+		       X(r2r_kind) kindx, X(r2r_kind) kindy, unsigned flags)
+{
+     int n[2];
+     X(r2r_kind) kind[2];
+     n[0] = nx;
+     n[1] = ny;
+     kind[0] = kindx;
+     kind[1] = kindy;
+     return X(plan_r2r)(2, n, in, out, kind, flags);
+}
diff --git a/src/fftw3/api/plan-r2r-3d.c b/src/fftw3/api/plan-r2r-3d.c
new file mode 100644
index 0000000..84ae651
--- /dev/null
+++ b/src/fftw3/api/plan-r2r-3d.c
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_r2r_3d)(int nx, int ny, int nz,
+		       R *in, R *out, X(r2r_kind) kindx,
+		       X(r2r_kind) kindy, X(r2r_kind) kindz, unsigned flags)
+{
+     int n[3];
+     X(r2r_kind) kind[3];
+     n[0] = nx;
+     n[1] = ny;
+     n[2] = nz;
+     kind[0] = kindx;
+     kind[1] = kindy;
+     kind[2] = kindz;
+     return X(plan_r2r)(3, n, in, out, kind, flags);
+}
diff --git a/src/fftw3/api/plan-r2r.c b/src/fftw3/api/plan-r2r.c
new file mode 100644
index 0000000..ba8f971
--- /dev/null
+++ b/src/fftw3/api/plan-r2r.c
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+X(plan) X(plan_r2r)(int rank, const int *n, R *in, R *out,
+		    const X(r2r_kind) * kind, unsigned flags)
+{
+     return X(plan_many_r2r)(rank, n, 1, in, 0, 1, 1, out, 0, 1, 1, kind,
+			     flags);
+}
diff --git a/src/fftw3/api/print-plan.c b/src/fftw3/api/print-plan.c
new file mode 100644
index 0000000..094446d
--- /dev/null
+++ b/src/fftw3/api/print-plan.c
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+void X(fprint_plan)(const X(plan) p, FILE *output_file)
+{
+     printer *pr = X(mkprinter_file)(output_file);
+     plan *pln = p->pln;
+     pln->adt->print(pln, pr);
+     X(printer_destroy)(pr);
+}
+
+void X(print_plan)(const X(plan) p)
+{
+     X(fprint_plan)(p, stdout);
+}
diff --git a/src/fftw3/api/rdft2-pad.c b/src/fftw3/api/rdft2-pad.c
new file mode 100644
index 0000000..499a55d
--- /dev/null
+++ b/src/fftw3/api/rdft2-pad.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include <string.h>
+#include "api.h"
+
+const int *X(rdft2_pad)(int rnk, const int *n, const int *nembed,
+			int inplace, int cmplx, int **nfree)
+{
+     A(FINITE_RNK(rnk));
+     *nfree = 0;
+     if (!nembed && rnk > 0) {
+          if (inplace || cmplx) {
+               int *np = (int *) MALLOC(sizeof(int) * rnk, PROBLEMS);
+               memcpy(np, n, sizeof(int) * rnk);
+               np[rnk - 1] = (n[rnk - 1] / 2 + 1) * (1 + !cmplx);
+               nembed = *nfree = np;
+          } else
+               nembed = n;
+     }
+     return nembed;
+}
diff --git a/src/fftw3/api/the-planner.c b/src/fftw3/api/the-planner.c
new file mode 100644
index 0000000..8f6d546
--- /dev/null
+++ b/src/fftw3/api/the-planner.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "api.h"
+
+static planner *plnr = 0;
+
+/* create the planner for the rest of the API */
+planner *X(the_planner)(void)
+{
+     if (!plnr) {
+          plnr = X(mkplanner)();
+          X(configure_planner)(plnr);
+     }
+
+     return plnr;
+}
+
+void X(cleanup)(void)
+{
+     if (plnr) {
+          X(planner_destroy)(plnr);
+          plnr = 0;
+     }
+}
diff --git a/src/fftw3/api/version.c b/src/fftw3/api/version.c
new file mode 100644
index 0000000..2abf1ec
--- /dev/null
+++ b/src/fftw3/api/version.c
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: version.c,v 1.1 2008/10/17 06:10:48 scuri Exp $ */
+
+#include "api.h"
+
+const char X(cc)[] = FFTW_CC;
+const char X(codelet_optim)[] = CODELET_OPTIM;
+
+const char X(version)[] = PACKAGE "-" VERSION
+
+#if HAVE_SSE
+   "-sse"
+#endif
+
+#if HAVE_SSE2
+   "-sse2"
+#endif
+
+#if HAVE_ALTIVEC
+   "-altivec"
+#endif
+
+#if HAVE_3DNOW
+   "-3dnow"
+#endif
+
+#if HAVE_K7
+   "-k7"
+#endif
+;
diff --git a/src/fftw3/api/x77.h b/src/fftw3/api/x77.h
new file mode 100644
index 0000000..74513b3
--- /dev/null
+++ b/src/fftw3/api/x77.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* Fortran-like (e.g. as in BLAS) type prefixes for F77 interface */
+#if defined(FFTW_SINGLE)
+#  define x77(name) CONCAT(sfftw_, name)
+#  define X77(NAME) CONCAT(SFFTW_, NAME)
+#elif defined(FFTW_LDOUBLE)
+/* FIXME: what is best?  BLAS uses D..._X, apparently.  Ugh. */
+#  define x77(name) CONCAT(lfftw_, name)
+#  define X77(NAME) CONCAT(LFFTW_, NAME)
+#else
+#  define x77(name) CONCAT(dfftw_, name)
+#  define X77(NAME) CONCAT(DFFTW_, NAME)
+#endif
diff --git a/src/fftw3/dft/buffered.c b/src/fftw3/dft/buffered.c
new file mode 100644
index 0000000..763ebe3
--- /dev/null
+++ b/src/fftw3/dft/buffered.c
@@ -0,0 +1,323 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: buffered.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+#include "dft.h"
+
+typedef struct {
+     int nbuf;
+     int maxbufsz;
+     int skew_alignment;
+     int skew;
+     const char *nam;
+} bufadt;
+
+typedef struct {
+     solver super;
+     const bufadt *adt;
+} S;
+
+typedef struct {
+     plan_dft super;
+
+     plan *cld, *cldcpy, *cldrest;
+     int n, vl, nbuf, bufdist;
+     int ivs, ovs;
+     int roffset, ioffset;
+
+     const S *slv;
+} P;
+
+/* transform a vector input with the help of bufs */
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int nbuf = ego->nbuf;
+     R *bufs = (R *)MALLOC(sizeof(R) * nbuf * ego->bufdist * 2, BUFFERS);
+
+     plan_dft *cld = (plan_dft *) ego->cld;
+     plan_dft *cldcpy = (plan_dft *) ego->cldcpy;
+     plan_dft *cldrest;
+     int i, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     int roffset = ego->roffset, ioffset = ego->ioffset;
+
+     /* note unsigned i:  the obvious statement
+
+          for (i = 0; i <= vl - nbuf; i += nbuf) 
+
+	is wrong */
+     for (i = nbuf; i <= vl; i += nbuf) {
+          /* transform to bufs: */
+          cld->apply((plan *) cld, ri, ii, bufs + roffset, bufs + ioffset);
+	  ri += ivs; ii += ivs;
+
+          /* copy back */
+          cldcpy->apply((plan *) cldcpy, bufs+roffset, bufs+ioffset, ro, io);
+	  ro += ovs; io += ovs;
+     }
+
+     /* Do the remaining transforms, if any: */
+     cldrest = (plan_dft *) ego->cldrest;
+     cldrest->apply((plan *) cldrest, ri, ii, ro, io);
+
+     X(ifree)(bufs);
+}
+
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+
+     AWAKE(ego->cld, flg);
+     AWAKE(ego->cldcpy, flg);
+     AWAKE(ego->cldrest, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cldrest);
+     X(plan_destroy_internal)(ego->cldcpy);
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(%s-%d%v/%d-%d%(%p%)%(%p%)%(%p%))",
+              ego->slv->adt->nam,
+              ego->n, ego->nbuf,
+              ego->vl, ego->bufdist % ego->n,
+              ego->cld, ego->cldcpy, ego->cldrest);
+}
+
+static int compute_nbuf(int n, int vl, const S *ego)
+{
+     return X(compute_nbuf)(n, vl, ego->adt->nbuf, ego->adt->maxbufsz);
+}
+
+static int toobig(int n, const S *ego)
+{
+     return (n > ego->adt->maxbufsz);
+}
+
+static int applicable0(const problem *p_, const S *ego, const planner *plnr)
+{
+     if (DFTP(p_)) {
+          const problem_dft *p = (const problem_dft *) p_;
+          const iodim *d = p->sz->dims;
+
+          if (1
+	      && p->vecsz->rnk <= 1
+	      && p->sz->rnk == 1
+	       ) {
+
+	       if (toobig(p->sz->dims[0].n, ego) && CONSERVE_MEMORYP(plnr))
+                    return 0;
+
+               /*
+		 In principle, the buffered transforms might be useful
+		 when working out of place.  However, in order to
+		 prevent infinite loops in the planner, we require
+		 that the output stride of the buffered transforms be
+		 greater than 2.
+               */
+               if (p->ri != p->ro)
+                    return (d[0].os > 2);
+
+               /* We can always do a single transform in-place */
+               if (p->vecsz->rnk == 0)
+                    return 1;
+
+               /*
+		* If the problem is in place, the input/output strides must
+		* be the same or the whole thing must fit in the buffer.
+		*/
+               return ((X(tensor_inplace_strides2)(p->sz, p->vecsz))
+                       || (compute_nbuf(d[0].n, p->vecsz->dims[0].n, ego)
+                           == p->vecsz->dims[0].n));
+          }
+     }
+     return 0;
+}
+
+static int applicable(const problem *p_, const S *ego, const planner *plnr)
+{
+     if (NO_BUFFERINGP(plnr)) return 0;
+     if (!applicable0(p_, ego, plnr)) return 0;
+
+     if (NO_UGLYP(plnr)) {
+	  const problem_dft *p = (const problem_dft *) p_;
+	  if (p->ri != p->ro) return 0;
+	  if (toobig(p->sz->dims[0].n, ego)) return 0;
+     }
+
+     return 1;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     const bufadt *adt = ego->adt;
+     P *pln;
+     plan *cld = (plan *) 0;
+     plan *cldcpy = (plan *) 0;
+     plan *cldrest = (plan *) 0;
+     const problem_dft *p = (const problem_dft *) p_;
+     R *bufs = (R *) 0;
+     int nbuf = 0, bufdist, n, vl;
+     int ivs, ovs, roffset, ioffset;
+
+     static const plan_adt padt = {
+	  X(dft_solve), awake, print, destroy
+     };
+
+     if (!applicable(p_, ego, plnr))
+          goto nada;
+
+     n = X(tensor_sz)(p->sz);
+
+     X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs);
+
+     nbuf = compute_nbuf(n, vl, ego);
+     A(nbuf > 0);
+
+     /*
+      * Determine BUFDIST, the offset between successive array bufs.
+      * bufdist = n + skew, where skew is chosen such that bufdist %
+      * skew_alignment = skew.
+      */
+     if (vl == 1) {
+          bufdist = n;
+     } else {
+          bufdist =
+               n + ((adt->skew_alignment + adt->skew - n % adt->skew_alignment)
+                    % adt->skew_alignment);
+          A(p->vecsz->rnk == 1);
+     }
+
+     /* attempt to keep real and imaginary part in the same order,
+	so as to allow optimizations in the the copy plan */
+     roffset = (p->ri - p->ii > 0) ? 1 : 0;
+     ioffset = 1 - roffset;
+
+     /* initial allocation for the purpose of planning */
+     bufs = (R *) MALLOC(sizeof(R) * nbuf * bufdist * 2, BUFFERS);
+
+     cld = X(mkplan_d)(plnr,
+		       X(mkproblem_dft_d)(
+			    X(mktensor_1d)(n, p->sz->dims[0].is, 2),
+			    X(mktensor_1d)(nbuf, ivs, bufdist * 2),
+			    TAINT(p->ri, ivs * nbuf),
+			    TAINT(p->ii, ivs * nbuf),
+			    bufs + roffset, 
+			    bufs + ioffset));
+     if (!cld)
+          goto nada;
+
+     /* copying back from the buffer is a rank-0 transform: */
+     cldcpy = X(mkplan_d)(plnr,
+			  X(mkproblem_dft_d)(
+			       X(mktensor_0d)(),
+			       X(mktensor_2d)(nbuf, bufdist * 2, ovs,
+					      n, 2, p->sz->dims[0].os),
+			       bufs + roffset, 
+			       bufs + ioffset, 
+			       TAINT(p->ro, ovs * nbuf), 
+			       TAINT(p->io, ovs * nbuf)));
+     if (!cldcpy)
+          goto nada;
+
+     /* deallocate buffers, let apply() allocate them for real */
+     X(ifree)(bufs);
+     bufs = 0;
+
+     /* plan the leftover transforms (cldrest): */
+     {
+	  int id = ivs * (nbuf * (vl / nbuf));
+	  int od = ovs * (nbuf * (vl / nbuf));
+	  cldrest = X(mkplan_d)(plnr, 
+				X(mkproblem_dft_d)(
+				     X(tensor_copy)(p->sz),
+				     X(mktensor_1d)(vl % nbuf, ivs, ovs),
+				     p->ri+id, p->ii+id, p->ro+od, p->io+od));
+     }
+     if (!cldrest)
+          goto nada;
+
+     pln = MKPLAN_DFT(P, &padt, apply);
+     pln->cld = cld;
+     pln->cldcpy = cldcpy;
+     pln->cldrest = cldrest;
+     pln->slv = ego;
+     pln->n = n;
+     pln->vl = vl;
+     pln->ivs = ivs * nbuf;
+     pln->ovs = ovs * nbuf;
+     pln->roffset = roffset;
+     pln->ioffset = ioffset;
+
+     pln->nbuf = nbuf;
+     pln->bufdist = bufdist;
+
+     {
+	  opcnt t;
+	  X(ops_add)(&cld->ops, &cldcpy->ops, &t);
+	  X(ops_madd)(vl / nbuf, &t, &cldrest->ops, &pln->super.super.ops);
+     }
+
+     return &(pln->super.super);
+
+ nada:
+     X(ifree0)(bufs);
+     X(plan_destroy_internal)(cldrest);
+     X(plan_destroy_internal)(cldcpy);
+     X(plan_destroy_internal)(cld);
+     return (plan *) 0;
+}
+
+static solver *mksolver(const bufadt *adt)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->adt = adt;
+     return &(slv->super);
+}
+
+
+void X(dft_buffered_register)(planner *p)
+{
+     /* FIXME: what are good defaults? */
+     static const bufadt adt = {
+	  /* nbuf */           8,
+	  /* maxbufsz */       (65536 / sizeof(R)),
+	  /* skew_alignment */ 8,
+#if HAVE_SIMD  /* 5 is odd and screws up the alignment. */
+	  /* skew */           6,
+#else
+	  /* skew */           5,
+#endif
+	  /* nam */            "dft-buffered"
+     };
+
+     REGISTER_SOLVER(p, mksolver(&adt));
+}
diff --git a/src/fftw3/dft/codelet-dft.h b/src/fftw3/dft/codelet-dft.h
new file mode 100644
index 0000000..1a5d376
--- /dev/null
+++ b/src/fftw3/dft/codelet-dft.h
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: codelet-dft.h,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/*
+ * This header file must include every file or define every
+ * type or macro which is required to compile a codelet.
+ */
+
+#ifndef __CODELET_H__
+#define __CODELET_H__
+
+#include "ifftw.h"
+
+/**************************************************************
+ * types of codelets
+ **************************************************************/
+
+/* DFT codelets */
+typedef struct kdft_desc_s kdft_desc;
+
+typedef struct {
+     int (*okp)(
+	  const kdft_desc *desc,
+	  const R *ri, const R *ii, const R *ro, const R *io,
+	  int is, int os, int vl, int ivs, int ovs,
+	  const planner *plnr);
+     int vl;
+} kdft_genus;
+
+struct kdft_desc_s {
+     int sz;    /* size of transform computed */
+     const char *nam;
+     opcnt ops;
+     const kdft_genus *genus;
+     int is;
+     int os;
+     int ivs;
+     int ovs;
+};
+
+typedef void (*kdft) (const R *ri, const R *ii, R *ro, R *io,
+                      stride is, stride os, int vl, int ivs, int ovs);
+void X(kdft_register)(planner *p, kdft codelet, const kdft_desc *desc);
+
+
+typedef struct ct_desc_s ct_desc;
+
+typedef struct {
+     int (*okp)(
+	  const struct ct_desc_s *desc,
+	  const R *rio, const R *iio, int ios, int vs, int m, int dist,
+	  const planner *plnr);
+     int vl;
+} ct_genus;
+
+struct ct_desc_s {
+     int radix;
+     const char *nam;
+     const tw_instr *tw;
+     opcnt ops;
+     const ct_genus *genus;
+     int s1;
+     int s2;
+     int dist;
+};
+
+typedef const R *(*kdft_dit) (R *rioarray, R *iioarray, const R *W,
+                              stride ios, int m, int dist);
+void X(kdft_dit_register)(planner *p, kdft_dit codelet, const ct_desc *desc);
+
+
+typedef const R *(*kdft_difsq) (R *rioarray, R *iioarray,
+                                const R *W, stride is, stride vs,
+                                int m, int dist);
+void X(kdft_difsq_register)(planner *p, kdft_difsq codelet,
+                            const ct_desc *desc);
+
+
+typedef const R *(*kdft_dif) (R *rioarray, R *iioarray, const R *W,
+                              stride ios, int m, int dist);
+void X(kdft_dif_register)(planner *p, kdft_dif codelet, const ct_desc *desc);
+
+extern const solvtab X(solvtab_dft_standard);
+extern const solvtab X(solvtab_dft_inplace);
+
+#if HAVE_K7
+extern const solvtab X(solvtab_dft_k7);
+#endif
+
+#if HAVE_SIMD
+extern const solvtab X(solvtab_dft_simd);
+#endif
+
+#endif				/* __CODELET_H__ */
diff --git a/src/fftw3/dft/codelets/f.h b/src/fftw3/dft/codelets/f.h
new file mode 100644
index 0000000..6a9dd0b
--- /dev/null
+++ b/src/fftw3/dft/codelets/f.h
@@ -0,0 +1 @@
+#include "t.h"  /* same stuff, no need to duplicate */
diff --git a/src/fftw3/dft/codelets/inplace/icodlist.c b/src/fftw3/dft/codelets/inplace/icodlist.c
new file mode 100644
index 0000000..708755e
--- /dev/null
+++ b/src/fftw3/dft/codelets/inplace/icodlist.c
@@ -0,0 +1,20 @@
+#include "ifftw.h"
+
+extern void X(codelet_q1_2)(planner *);
+extern void X(codelet_q1_4)(planner *);
+extern void X(codelet_q1_8)(planner *);
+extern void X(codelet_q1_3)(planner *);
+extern void X(codelet_q1_5)(planner *);
+extern void X(codelet_q1_6)(planner *);
+
+
+extern const solvtab X(solvtab_dft_inplace);
+const solvtab X(solvtab_dft_inplace) = {
+   SOLVTAB(X(codelet_q1_2)),
+   SOLVTAB(X(codelet_q1_4)),
+   SOLVTAB(X(codelet_q1_8)),
+   SOLVTAB(X(codelet_q1_3)),
+   SOLVTAB(X(codelet_q1_5)),
+   SOLVTAB(X(codelet_q1_6)),
+   SOLVTAB_END
+};
diff --git a/src/fftw3/dft/codelets/inplace/q1_2.c b/src/fftw3/dft/codelets/inplace/q1_2.c
new file mode 100644
index 0000000..6057fe6
--- /dev/null
+++ b/src/fftw3/dft/codelets/inplace/q1_2.c
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:39:14 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twidsq -compact -variables 4 -reload-twiddle -dif -n 2 -name q1_2 -include q.h */
+
+/*
+ * This function contains 12 FP additions, 8 FP multiplications,
+ * (or, 8 additions, 4 multiplications, 4 fused multiply/add),
+ * 17 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: q1_2.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_2.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_2.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ */
+
+#include "q.h"
+
+static const R *q1_2(R *rio, R *iio, const R *W, stride is, stride vs, int m, int dist)
+{
+     int i;
+     for (i = m; i > 0; i = i - 1, rio = rio + dist, iio = iio + dist, W = W + 2) {
+	  E T1, T2, T4, T6, T7, T8, T9, Ta, Tc, Te, Tf, Tg;
+	  T1 = rio[0];
+	  T2 = rio[WS(is, 1)];
+	  T4 = T1 - T2;
+	  T6 = iio[0];
+	  T7 = iio[WS(is, 1)];
+	  T8 = T6 - T7;
+	  T9 = rio[WS(vs, 1)];
+	  Ta = rio[WS(vs, 1) + WS(is, 1)];
+	  Tc = T9 - Ta;
+	  Te = iio[WS(vs, 1)];
+	  Tf = iio[WS(vs, 1) + WS(is, 1)];
+	  Tg = Te - Tf;
+	  rio[0] = T1 + T2;
+	  iio[0] = T6 + T7;
+	  rio[WS(is, 1)] = T9 + Ta;
+	  iio[WS(is, 1)] = Te + Tf;
+	  {
+	       E Tb, Td, T3, T5;
+	       Tb = W[0];
+	       Td = W[1];
+	       rio[WS(vs, 1) + WS(is, 1)] = FMA(Tb, Tc, Td * Tg);
+	       iio[WS(vs, 1) + WS(is, 1)] = FNMS(Td, Tc, Tb * Tg);
+	       T3 = W[0];
+	       T5 = W[1];
+	       rio[WS(vs, 1)] = FMA(T3, T4, T5 * T8);
+	       iio[WS(vs, 1)] = FNMS(T5, T4, T3 * T8);
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 2},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 2, "q1_2", twinstr, {8, 4, 4, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_q1_2) (planner *p) {
+     X(kdft_difsq_register) (p, q1_2, &desc);
+}
diff --git a/src/fftw3/dft/codelets/inplace/q1_3.c b/src/fftw3/dft/codelets/inplace/q1_3.c
new file mode 100644
index 0000000..a05996f
--- /dev/null
+++ b/src/fftw3/dft/codelets/inplace/q1_3.c
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:39:14 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twidsq -compact -variables 4 -reload-twiddle -dif -n 3 -name q1_3 -include q.h */
+
+/*
+ * This function contains 48 FP additions, 36 FP multiplications,
+ * (or, 30 additions, 18 multiplications, 18 fused multiply/add),
+ * 35 stack variables, and 36 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: q1_3.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_3.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_3.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ */
+
+#include "q.h"
+
+static const R *q1_3(R *rio, R *iio, const R *W, stride is, stride vs, int m, int dist)
+{
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = m; i > 0; i = i - 1, rio = rio + dist, iio = iio + dist, W = W + 4) {
+	  E T1, T4, T6, Tc, Td, Te, T9, Tf, Tl, To, Tq, Tw, Tx, Ty, Tt;
+	  E Tz, TR, TS, TN, TT, TF, TI, TK, TQ;
+	  {
+	       E T2, T3, Tr, Ts;
+	       T1 = rio[0];
+	       T2 = rio[WS(is, 1)];
+	       T3 = rio[WS(is, 2)];
+	       T4 = T2 + T3;
+	       T6 = FNMS(KP500000000, T4, T1);
+	       Tc = KP866025403 * (T3 - T2);
+	       {
+		    E T7, T8, Tm, Tn;
+		    Td = iio[0];
+		    T7 = iio[WS(is, 1)];
+		    T8 = iio[WS(is, 2)];
+		    Te = T7 + T8;
+		    T9 = KP866025403 * (T7 - T8);
+		    Tf = FNMS(KP500000000, Te, Td);
+		    Tl = rio[WS(vs, 1)];
+		    Tm = rio[WS(vs, 1) + WS(is, 1)];
+		    Tn = rio[WS(vs, 1) + WS(is, 2)];
+		    To = Tm + Tn;
+		    Tq = FNMS(KP500000000, To, Tl);
+		    Tw = KP866025403 * (Tn - Tm);
+	       }
+	       Tx = iio[WS(vs, 1)];
+	       Tr = iio[WS(vs, 1) + WS(is, 1)];
+	       Ts = iio[WS(vs, 1) + WS(is, 2)];
+	       Ty = Tr + Ts;
+	       Tt = KP866025403 * (Tr - Ts);
+	       Tz = FNMS(KP500000000, Ty, Tx);
+	       {
+		    E TL, TM, TG, TH;
+		    TR = iio[WS(vs, 2)];
+		    TL = iio[WS(vs, 2) + WS(is, 1)];
+		    TM = iio[WS(vs, 2) + WS(is, 2)];
+		    TS = TL + TM;
+		    TN = KP866025403 * (TL - TM);
+		    TT = FNMS(KP500000000, TS, TR);
+		    TF = rio[WS(vs, 2)];
+		    TG = rio[WS(vs, 2) + WS(is, 1)];
+		    TH = rio[WS(vs, 2) + WS(is, 2)];
+		    TI = TG + TH;
+		    TK = FNMS(KP500000000, TI, TF);
+		    TQ = KP866025403 * (TH - TG);
+	       }
+	  }
+	  rio[0] = T1 + T4;
+	  iio[0] = Td + Te;
+	  rio[WS(is, 1)] = Tl + To;
+	  iio[WS(is, 1)] = Tx + Ty;
+	  iio[WS(is, 2)] = TR + TS;
+	  rio[WS(is, 2)] = TF + TI;
+	  {
+	       E Ta, Tg, T5, Tb;
+	       Ta = T6 + T9;
+	       Tg = Tc + Tf;
+	       T5 = W[0];
+	       Tb = W[1];
+	       rio[WS(vs, 1)] = FMA(T5, Ta, Tb * Tg);
+	       iio[WS(vs, 1)] = FNMS(Tb, Ta, T5 * Tg);
+	  }
+	  {
+	       E TW, TY, TV, TX;
+	       TW = TK - TN;
+	       TY = TT - TQ;
+	       TV = W[2];
+	       TX = W[3];
+	       rio[WS(vs, 2) + WS(is, 2)] = FMA(TV, TW, TX * TY);
+	       iio[WS(vs, 2) + WS(is, 2)] = FNMS(TX, TW, TV * TY);
+	  }
+	  {
+	       E TC, TE, TB, TD;
+	       TC = Tq - Tt;
+	       TE = Tz - Tw;
+	       TB = W[2];
+	       TD = W[3];
+	       rio[WS(vs, 2) + WS(is, 1)] = FMA(TB, TC, TD * TE);
+	       iio[WS(vs, 2) + WS(is, 1)] = FNMS(TD, TC, TB * TE);
+	  }
+	  {
+	       E Tu, TA, Tp, Tv;
+	       Tu = Tq + Tt;
+	       TA = Tw + Tz;
+	       Tp = W[0];
+	       Tv = W[1];
+	       rio[WS(vs, 1) + WS(is, 1)] = FMA(Tp, Tu, Tv * TA);
+	       iio[WS(vs, 1) + WS(is, 1)] = FNMS(Tv, Tu, Tp * TA);
+	  }
+	  {
+	       E TO, TU, TJ, TP;
+	       TO = TK + TN;
+	       TU = TQ + TT;
+	       TJ = W[0];
+	       TP = W[1];
+	       rio[WS(vs, 1) + WS(is, 2)] = FMA(TJ, TO, TP * TU);
+	       iio[WS(vs, 1) + WS(is, 2)] = FNMS(TP, TO, TJ * TU);
+	  }
+	  {
+	       E Ti, Tk, Th, Tj;
+	       Ti = T6 - T9;
+	       Tk = Tf - Tc;
+	       Th = W[2];
+	       Tj = W[3];
+	       rio[WS(vs, 2)] = FMA(Th, Ti, Tj * Tk);
+	       iio[WS(vs, 2)] = FNMS(Tj, Ti, Th * Tk);
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 3},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 3, "q1_3", twinstr, {30, 18, 18, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_q1_3) (planner *p) {
+     X(kdft_difsq_register) (p, q1_3, &desc);
+}
diff --git a/src/fftw3/dft/codelets/inplace/q1_4.c b/src/fftw3/dft/codelets/inplace/q1_4.c
new file mode 100644
index 0000000..c4300f5
--- /dev/null
+++ b/src/fftw3/dft/codelets/inplace/q1_4.c
@@ -0,0 +1,266 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:39:14 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twidsq -compact -variables 4 -reload-twiddle -dif -n 4 -name q1_4 -include q.h */
+
+/*
+ * This function contains 88 FP additions, 48 FP multiplications,
+ * (or, 64 additions, 24 multiplications, 24 fused multiply/add),
+ * 37 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: q1_4.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_4.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_4.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ */
+
+#include "q.h"
+
+static const R *q1_4(R *rio, R *iio, const R *W, stride is, stride vs, int m, int dist)
+{
+     int i;
+     for (i = m; i > 0; i = i - 1, rio = rio + dist, iio = iio + dist, W = W + 6) {
+	  E T3, Te, Tb, Tq, T6, T8, Th, Tr, Tv, TG, TD, TS, Ty, TA, TJ;
+	  E TT, TX, T18, T15, T1k, T10, T12, T1b, T1l, T1p, T1A, T1x, T1M, T1s, T1u;
+	  E T1D, T1N;
+	  {
+	       E T1, T2, T9, Ta;
+	       T1 = rio[0];
+	       T2 = rio[WS(is, 2)];
+	       T3 = T1 + T2;
+	       Te = T1 - T2;
+	       T9 = iio[0];
+	       Ta = iio[WS(is, 2)];
+	       Tb = T9 - Ta;
+	       Tq = T9 + Ta;
+	  }
+	  {
+	       E T4, T5, Tf, Tg;
+	       T4 = rio[WS(is, 1)];
+	       T5 = rio[WS(is, 3)];
+	       T6 = T4 + T5;
+	       T8 = T4 - T5;
+	       Tf = iio[WS(is, 1)];
+	       Tg = iio[WS(is, 3)];
+	       Th = Tf - Tg;
+	       Tr = Tf + Tg;
+	  }
+	  {
+	       E Tt, Tu, TB, TC;
+	       Tt = rio[WS(vs, 1)];
+	       Tu = rio[WS(vs, 1) + WS(is, 2)];
+	       Tv = Tt + Tu;
+	       TG = Tt - Tu;
+	       TB = iio[WS(vs, 1)];
+	       TC = iio[WS(vs, 1) + WS(is, 2)];
+	       TD = TB - TC;
+	       TS = TB + TC;
+	  }
+	  {
+	       E Tw, Tx, TH, TI;
+	       Tw = rio[WS(vs, 1) + WS(is, 1)];
+	       Tx = rio[WS(vs, 1) + WS(is, 3)];
+	       Ty = Tw + Tx;
+	       TA = Tw - Tx;
+	       TH = iio[WS(vs, 1) + WS(is, 1)];
+	       TI = iio[WS(vs, 1) + WS(is, 3)];
+	       TJ = TH - TI;
+	       TT = TH + TI;
+	  }
+	  {
+	       E TV, TW, T13, T14;
+	       TV = rio[WS(vs, 2)];
+	       TW = rio[WS(vs, 2) + WS(is, 2)];
+	       TX = TV + TW;
+	       T18 = TV - TW;
+	       T13 = iio[WS(vs, 2)];
+	       T14 = iio[WS(vs, 2) + WS(is, 2)];
+	       T15 = T13 - T14;
+	       T1k = T13 + T14;
+	  }
+	  {
+	       E TY, TZ, T19, T1a;
+	       TY = rio[WS(vs, 2) + WS(is, 1)];
+	       TZ = rio[WS(vs, 2) + WS(is, 3)];
+	       T10 = TY + TZ;
+	       T12 = TY - TZ;
+	       T19 = iio[WS(vs, 2) + WS(is, 1)];
+	       T1a = iio[WS(vs, 2) + WS(is, 3)];
+	       T1b = T19 - T1a;
+	       T1l = T19 + T1a;
+	  }
+	  {
+	       E T1n, T1o, T1v, T1w;
+	       T1n = rio[WS(vs, 3)];
+	       T1o = rio[WS(vs, 3) + WS(is, 2)];
+	       T1p = T1n + T1o;
+	       T1A = T1n - T1o;
+	       T1v = iio[WS(vs, 3)];
+	       T1w = iio[WS(vs, 3) + WS(is, 2)];
+	       T1x = T1v - T1w;
+	       T1M = T1v + T1w;
+	  }
+	  {
+	       E T1q, T1r, T1B, T1C;
+	       T1q = rio[WS(vs, 3) + WS(is, 1)];
+	       T1r = rio[WS(vs, 3) + WS(is, 3)];
+	       T1s = T1q + T1r;
+	       T1u = T1q - T1r;
+	       T1B = iio[WS(vs, 3) + WS(is, 1)];
+	       T1C = iio[WS(vs, 3) + WS(is, 3)];
+	       T1D = T1B - T1C;
+	       T1N = T1B + T1C;
+	  }
+	  rio[0] = T3 + T6;
+	  iio[0] = Tq + Tr;
+	  rio[WS(is, 1)] = Tv + Ty;
+	  iio[WS(is, 1)] = TS + TT;
+	  rio[WS(is, 2)] = TX + T10;
+	  iio[WS(is, 2)] = T1k + T1l;
+	  iio[WS(is, 3)] = T1M + T1N;
+	  rio[WS(is, 3)] = T1p + T1s;
+	  {
+	       E Tc, Ti, T7, Td;
+	       Tc = T8 + Tb;
+	       Ti = Te - Th;
+	       T7 = W[4];
+	       Td = W[5];
+	       iio[WS(vs, 3)] = FNMS(Td, Ti, T7 * Tc);
+	       rio[WS(vs, 3)] = FMA(Td, Tc, T7 * Ti);
+	  }
+	  {
+	       E T1K, T1O, T1J, T1L;
+	       T1K = T1p - T1s;
+	       T1O = T1M - T1N;
+	       T1J = W[2];
+	       T1L = W[3];
+	       rio[WS(vs, 2) + WS(is, 3)] = FMA(T1J, T1K, T1L * T1O);
+	       iio[WS(vs, 2) + WS(is, 3)] = FNMS(T1L, T1K, T1J * T1O);
+	  }
+	  {
+	       E Tk, Tm, Tj, Tl;
+	       Tk = Tb - T8;
+	       Tm = Te + Th;
+	       Tj = W[0];
+	       Tl = W[1];
+	       iio[WS(vs, 1)] = FNMS(Tl, Tm, Tj * Tk);
+	       rio[WS(vs, 1)] = FMA(Tl, Tk, Tj * Tm);
+	  }
+	  {
+	       E To, Ts, Tn, Tp;
+	       To = T3 - T6;
+	       Ts = Tq - Tr;
+	       Tn = W[2];
+	       Tp = W[3];
+	       rio[WS(vs, 2)] = FMA(Tn, To, Tp * Ts);
+	       iio[WS(vs, 2)] = FNMS(Tp, To, Tn * Ts);
+	  }
+	  {
+	       E T16, T1c, T11, T17;
+	       T16 = T12 + T15;
+	       T1c = T18 - T1b;
+	       T11 = W[4];
+	       T17 = W[5];
+	       iio[WS(vs, 3) + WS(is, 2)] = FNMS(T17, T1c, T11 * T16);
+	       rio[WS(vs, 3) + WS(is, 2)] = FMA(T17, T16, T11 * T1c);
+	  }
+	  {
+	       E T1G, T1I, T1F, T1H;
+	       T1G = T1x - T1u;
+	       T1I = T1A + T1D;
+	       T1F = W[0];
+	       T1H = W[1];
+	       iio[WS(vs, 1) + WS(is, 3)] = FNMS(T1H, T1I, T1F * T1G);
+	       rio[WS(vs, 1) + WS(is, 3)] = FMA(T1H, T1G, T1F * T1I);
+	  }
+	  {
+	       E TQ, TU, TP, TR;
+	       TQ = Tv - Ty;
+	       TU = TS - TT;
+	       TP = W[2];
+	       TR = W[3];
+	       rio[WS(vs, 2) + WS(is, 1)] = FMA(TP, TQ, TR * TU);
+	       iio[WS(vs, 2) + WS(is, 1)] = FNMS(TR, TQ, TP * TU);
+	  }
+	  {
+	       E T1e, T1g, T1d, T1f;
+	       T1e = T15 - T12;
+	       T1g = T18 + T1b;
+	       T1d = W[0];
+	       T1f = W[1];
+	       iio[WS(vs, 1) + WS(is, 2)] = FNMS(T1f, T1g, T1d * T1e);
+	       rio[WS(vs, 1) + WS(is, 2)] = FMA(T1f, T1e, T1d * T1g);
+	  }
+	  {
+	       E T1i, T1m, T1h, T1j;
+	       T1i = TX - T10;
+	       T1m = T1k - T1l;
+	       T1h = W[2];
+	       T1j = W[3];
+	       rio[WS(vs, 2) + WS(is, 2)] = FMA(T1h, T1i, T1j * T1m);
+	       iio[WS(vs, 2) + WS(is, 2)] = FNMS(T1j, T1i, T1h * T1m);
+	  }
+	  {
+	       E T1y, T1E, T1t, T1z;
+	       T1y = T1u + T1x;
+	       T1E = T1A - T1D;
+	       T1t = W[4];
+	       T1z = W[5];
+	       iio[WS(vs, 3) + WS(is, 3)] = FNMS(T1z, T1E, T1t * T1y);
+	       rio[WS(vs, 3) + WS(is, 3)] = FMA(T1z, T1y, T1t * T1E);
+	  }
+	  {
+	       E TM, TO, TL, TN;
+	       TM = TD - TA;
+	       TO = TG + TJ;
+	       TL = W[0];
+	       TN = W[1];
+	       iio[WS(vs, 1) + WS(is, 1)] = FNMS(TN, TO, TL * TM);
+	       rio[WS(vs, 1) + WS(is, 1)] = FMA(TN, TM, TL * TO);
+	  }
+	  {
+	       E TE, TK, Tz, TF;
+	       TE = TA + TD;
+	       TK = TG - TJ;
+	       Tz = W[4];
+	       TF = W[5];
+	       iio[WS(vs, 3) + WS(is, 1)] = FNMS(TF, TK, Tz * TE);
+	       rio[WS(vs, 3) + WS(is, 1)] = FMA(TF, TE, Tz * TK);
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 4},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 4, "q1_4", twinstr, {64, 24, 24, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_q1_4) (planner *p) {
+     X(kdft_difsq_register) (p, q1_4, &desc);
+}
diff --git a/src/fftw3/dft/codelets/inplace/q1_5.c b/src/fftw3/dft/codelets/inplace/q1_5.c
new file mode 100644
index 0000000..144dcb8
--- /dev/null
+++ b/src/fftw3/dft/codelets/inplace/q1_5.c
@@ -0,0 +1,477 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:39:14 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twidsq -compact -variables 4 -reload-twiddle -dif -n 5 -name q1_5 -include q.h */
+
+/*
+ * This function contains 200 FP additions, 140 FP multiplications,
+ * (or, 130 additions, 70 multiplications, 70 fused multiply/add),
+ * 75 stack variables, and 100 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: q1_5.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_5.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_5.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ */
+
+#include "q.h"
+
+static const R *q1_5(R *rio, R *iio, const R *W, stride is, stride vs, int m, int dist)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     int i;
+     for (i = m; i > 0; i = i - 1, rio = rio + dist, iio = iio + dist, W = W + 8) {
+	  E T1, Ta, TG, Tv, T8, Tb, Tp, Tj, TD, To, Tq, Tr, TN, TW, T1s;
+	  E T1h, TU, TX, T1b, T15, T1p, T1a, T1c, T1d, T1z, T1I, T2e, T23, T1G, T1J;
+	  E T1X, T1R, T2b, T1W, T1Y, T1Z, T3v, T3p, T3J, T3u, T3w, T3x, T37, T3g, T3M;
+	  E T3B, T3e, T3h, T2l, T2u, T30, T2P, T2s, T2v, T2J, T2D, T2X, T2I, T2K, T2L;
+	  {
+	       E T7, Tu, T4, Tt;
+	       T1 = rio[0];
+	       {
+		    E T5, T6, T2, T3;
+		    T5 = rio[WS(is, 2)];
+		    T6 = rio[WS(is, 3)];
+		    T7 = T5 + T6;
+		    Tu = T5 - T6;
+		    T2 = rio[WS(is, 1)];
+		    T3 = rio[WS(is, 4)];
+		    T4 = T2 + T3;
+		    Tt = T2 - T3;
+	       }
+	       Ta = KP559016994 * (T4 - T7);
+	       TG = FNMS(KP587785252, Tt, KP951056516 * Tu);
+	       Tv = FMA(KP951056516, Tt, KP587785252 * Tu);
+	       T8 = T4 + T7;
+	       Tb = FNMS(KP250000000, T8, T1);
+	  }
+	  {
+	       E Ti, Tn, Tf, Tm;
+	       Tp = iio[0];
+	       {
+		    E Tg, Th, Td, Te;
+		    Tg = iio[WS(is, 2)];
+		    Th = iio[WS(is, 3)];
+		    Ti = Tg - Th;
+		    Tn = Tg + Th;
+		    Td = iio[WS(is, 1)];
+		    Te = iio[WS(is, 4)];
+		    Tf = Td - Te;
+		    Tm = Td + Te;
+	       }
+	       Tj = FMA(KP951056516, Tf, KP587785252 * Ti);
+	       TD = FNMS(KP587785252, Tf, KP951056516 * Ti);
+	       To = KP559016994 * (Tm - Tn);
+	       Tq = Tm + Tn;
+	       Tr = FNMS(KP250000000, Tq, Tp);
+	  }
+	  {
+	       E TT, T1g, TQ, T1f;
+	       TN = rio[WS(vs, 1)];
+	       {
+		    E TR, TS, TO, TP;
+		    TR = rio[WS(vs, 1) + WS(is, 2)];
+		    TS = rio[WS(vs, 1) + WS(is, 3)];
+		    TT = TR + TS;
+		    T1g = TR - TS;
+		    TO = rio[WS(vs, 1) + WS(is, 1)];
+		    TP = rio[WS(vs, 1) + WS(is, 4)];
+		    TQ = TO + TP;
+		    T1f = TO - TP;
+	       }
+	       TW = KP559016994 * (TQ - TT);
+	       T1s = FNMS(KP587785252, T1f, KP951056516 * T1g);
+	       T1h = FMA(KP951056516, T1f, KP587785252 * T1g);
+	       TU = TQ + TT;
+	       TX = FNMS(KP250000000, TU, TN);
+	  }
+	  {
+	       E T14, T19, T11, T18;
+	       T1b = iio[WS(vs, 1)];
+	       {
+		    E T12, T13, TZ, T10;
+		    T12 = iio[WS(vs, 1) + WS(is, 2)];
+		    T13 = iio[WS(vs, 1) + WS(is, 3)];
+		    T14 = T12 - T13;
+		    T19 = T12 + T13;
+		    TZ = iio[WS(vs, 1) + WS(is, 1)];
+		    T10 = iio[WS(vs, 1) + WS(is, 4)];
+		    T11 = TZ - T10;
+		    T18 = TZ + T10;
+	       }
+	       T15 = FMA(KP951056516, T11, KP587785252 * T14);
+	       T1p = FNMS(KP587785252, T11, KP951056516 * T14);
+	       T1a = KP559016994 * (T18 - T19);
+	       T1c = T18 + T19;
+	       T1d = FNMS(KP250000000, T1c, T1b);
+	  }
+	  {
+	       E T1F, T22, T1C, T21;
+	       T1z = rio[WS(vs, 2)];
+	       {
+		    E T1D, T1E, T1A, T1B;
+		    T1D = rio[WS(vs, 2) + WS(is, 2)];
+		    T1E = rio[WS(vs, 2) + WS(is, 3)];
+		    T1F = T1D + T1E;
+		    T22 = T1D - T1E;
+		    T1A = rio[WS(vs, 2) + WS(is, 1)];
+		    T1B = rio[WS(vs, 2) + WS(is, 4)];
+		    T1C = T1A + T1B;
+		    T21 = T1A - T1B;
+	       }
+	       T1I = KP559016994 * (T1C - T1F);
+	       T2e = FNMS(KP587785252, T21, KP951056516 * T22);
+	       T23 = FMA(KP951056516, T21, KP587785252 * T22);
+	       T1G = T1C + T1F;
+	       T1J = FNMS(KP250000000, T1G, T1z);
+	  }
+	  {
+	       E T1Q, T1V, T1N, T1U;
+	       T1X = iio[WS(vs, 2)];
+	       {
+		    E T1O, T1P, T1L, T1M;
+		    T1O = iio[WS(vs, 2) + WS(is, 2)];
+		    T1P = iio[WS(vs, 2) + WS(is, 3)];
+		    T1Q = T1O - T1P;
+		    T1V = T1O + T1P;
+		    T1L = iio[WS(vs, 2) + WS(is, 1)];
+		    T1M = iio[WS(vs, 2) + WS(is, 4)];
+		    T1N = T1L - T1M;
+		    T1U = T1L + T1M;
+	       }
+	       T1R = FMA(KP951056516, T1N, KP587785252 * T1Q);
+	       T2b = FNMS(KP587785252, T1N, KP951056516 * T1Q);
+	       T1W = KP559016994 * (T1U - T1V);
+	       T1Y = T1U + T1V;
+	       T1Z = FNMS(KP250000000, T1Y, T1X);
+	  }
+	  {
+	       E T3o, T3t, T3l, T3s;
+	       T3v = iio[WS(vs, 4)];
+	       {
+		    E T3m, T3n, T3j, T3k;
+		    T3m = iio[WS(vs, 4) + WS(is, 2)];
+		    T3n = iio[WS(vs, 4) + WS(is, 3)];
+		    T3o = T3m - T3n;
+		    T3t = T3m + T3n;
+		    T3j = iio[WS(vs, 4) + WS(is, 1)];
+		    T3k = iio[WS(vs, 4) + WS(is, 4)];
+		    T3l = T3j - T3k;
+		    T3s = T3j + T3k;
+	       }
+	       T3p = FMA(KP951056516, T3l, KP587785252 * T3o);
+	       T3J = FNMS(KP587785252, T3l, KP951056516 * T3o);
+	       T3u = KP559016994 * (T3s - T3t);
+	       T3w = T3s + T3t;
+	       T3x = FNMS(KP250000000, T3w, T3v);
+	  }
+	  {
+	       E T3d, T3A, T3a, T3z;
+	       T37 = rio[WS(vs, 4)];
+	       {
+		    E T3b, T3c, T38, T39;
+		    T3b = rio[WS(vs, 4) + WS(is, 2)];
+		    T3c = rio[WS(vs, 4) + WS(is, 3)];
+		    T3d = T3b + T3c;
+		    T3A = T3b - T3c;
+		    T38 = rio[WS(vs, 4) + WS(is, 1)];
+		    T39 = rio[WS(vs, 4) + WS(is, 4)];
+		    T3a = T38 + T39;
+		    T3z = T38 - T39;
+	       }
+	       T3g = KP559016994 * (T3a - T3d);
+	       T3M = FNMS(KP587785252, T3z, KP951056516 * T3A);
+	       T3B = FMA(KP951056516, T3z, KP587785252 * T3A);
+	       T3e = T3a + T3d;
+	       T3h = FNMS(KP250000000, T3e, T37);
+	  }
+	  {
+	       E T2r, T2O, T2o, T2N;
+	       T2l = rio[WS(vs, 3)];
+	       {
+		    E T2p, T2q, T2m, T2n;
+		    T2p = rio[WS(vs, 3) + WS(is, 2)];
+		    T2q = rio[WS(vs, 3) + WS(is, 3)];
+		    T2r = T2p + T2q;
+		    T2O = T2p - T2q;
+		    T2m = rio[WS(vs, 3) + WS(is, 1)];
+		    T2n = rio[WS(vs, 3) + WS(is, 4)];
+		    T2o = T2m + T2n;
+		    T2N = T2m - T2n;
+	       }
+	       T2u = KP559016994 * (T2o - T2r);
+	       T30 = FNMS(KP587785252, T2N, KP951056516 * T2O);
+	       T2P = FMA(KP951056516, T2N, KP587785252 * T2O);
+	       T2s = T2o + T2r;
+	       T2v = FNMS(KP250000000, T2s, T2l);
+	  }
+	  {
+	       E T2C, T2H, T2z, T2G;
+	       T2J = iio[WS(vs, 3)];
+	       {
+		    E T2A, T2B, T2x, T2y;
+		    T2A = iio[WS(vs, 3) + WS(is, 2)];
+		    T2B = iio[WS(vs, 3) + WS(is, 3)];
+		    T2C = T2A - T2B;
+		    T2H = T2A + T2B;
+		    T2x = iio[WS(vs, 3) + WS(is, 1)];
+		    T2y = iio[WS(vs, 3) + WS(is, 4)];
+		    T2z = T2x - T2y;
+		    T2G = T2x + T2y;
+	       }
+	       T2D = FMA(KP951056516, T2z, KP587785252 * T2C);
+	       T2X = FNMS(KP587785252, T2z, KP951056516 * T2C);
+	       T2I = KP559016994 * (T2G - T2H);
+	       T2K = T2G + T2H;
+	       T2L = FNMS(KP250000000, T2K, T2J);
+	  }
+	  rio[0] = T1 + T8;
+	  iio[0] = Tp + Tq;
+	  rio[WS(is, 1)] = TN + TU;
+	  iio[WS(is, 1)] = T1b + T1c;
+	  rio[WS(is, 2)] = T1z + T1G;
+	  iio[WS(is, 2)] = T1X + T1Y;
+	  iio[WS(is, 4)] = T3v + T3w;
+	  rio[WS(is, 4)] = T37 + T3e;
+	  rio[WS(is, 3)] = T2l + T2s;
+	  iio[WS(is, 3)] = T2J + T2K;
+	  {
+	       E Tk, Ty, Tw, TA, Tc, Ts;
+	       Tc = Ta + Tb;
+	       Tk = Tc + Tj;
+	       Ty = Tc - Tj;
+	       Ts = To + Tr;
+	       Tw = Ts - Tv;
+	       TA = Tv + Ts;
+	       {
+		    E T9, Tl, Tx, Tz;
+		    T9 = W[0];
+		    Tl = W[1];
+		    rio[WS(vs, 1)] = FMA(T9, Tk, Tl * Tw);
+		    iio[WS(vs, 1)] = FNMS(Tl, Tk, T9 * Tw);
+		    Tx = W[6];
+		    Tz = W[7];
+		    rio[WS(vs, 4)] = FMA(Tx, Ty, Tz * TA);
+		    iio[WS(vs, 4)] = FNMS(Tz, Ty, Tx * TA);
+	       }
+	  }
+	  {
+	       E TE, TK, TI, TM, TC, TH;
+	       TC = Tb - Ta;
+	       TE = TC - TD;
+	       TK = TC + TD;
+	       TH = Tr - To;
+	       TI = TG + TH;
+	       TM = TH - TG;
+	       {
+		    E TB, TF, TJ, TL;
+		    TB = W[2];
+		    TF = W[3];
+		    rio[WS(vs, 2)] = FMA(TB, TE, TF * TI);
+		    iio[WS(vs, 2)] = FNMS(TF, TE, TB * TI);
+		    TJ = W[4];
+		    TL = W[5];
+		    rio[WS(vs, 3)] = FMA(TJ, TK, TL * TM);
+		    iio[WS(vs, 3)] = FNMS(TL, TK, TJ * TM);
+	       }
+	  }
+	  {
+	       E T2c, T2i, T2g, T2k, T2a, T2f;
+	       T2a = T1J - T1I;
+	       T2c = T2a - T2b;
+	       T2i = T2a + T2b;
+	       T2f = T1Z - T1W;
+	       T2g = T2e + T2f;
+	       T2k = T2f - T2e;
+	       {
+		    E T29, T2d, T2h, T2j;
+		    T29 = W[2];
+		    T2d = W[3];
+		    rio[WS(vs, 2) + WS(is, 2)] = FMA(T29, T2c, T2d * T2g);
+		    iio[WS(vs, 2) + WS(is, 2)] = FNMS(T2d, T2c, T29 * T2g);
+		    T2h = W[4];
+		    T2j = W[5];
+		    rio[WS(vs, 3) + WS(is, 2)] = FMA(T2h, T2i, T2j * T2k);
+		    iio[WS(vs, 3) + WS(is, 2)] = FNMS(T2j, T2i, T2h * T2k);
+	       }
+	  }
+	  {
+	       E T3K, T3Q, T3O, T3S, T3I, T3N;
+	       T3I = T3h - T3g;
+	       T3K = T3I - T3J;
+	       T3Q = T3I + T3J;
+	       T3N = T3x - T3u;
+	       T3O = T3M + T3N;
+	       T3S = T3N - T3M;
+	       {
+		    E T3H, T3L, T3P, T3R;
+		    T3H = W[2];
+		    T3L = W[3];
+		    rio[WS(vs, 2) + WS(is, 4)] = FMA(T3H, T3K, T3L * T3O);
+		    iio[WS(vs, 2) + WS(is, 4)] = FNMS(T3L, T3K, T3H * T3O);
+		    T3P = W[4];
+		    T3R = W[5];
+		    rio[WS(vs, 3) + WS(is, 4)] = FMA(T3P, T3Q, T3R * T3S);
+		    iio[WS(vs, 3) + WS(is, 4)] = FNMS(T3R, T3Q, T3P * T3S);
+	       }
+	  }
+	  {
+	       E T1S, T26, T24, T28, T1K, T20;
+	       T1K = T1I + T1J;
+	       T1S = T1K + T1R;
+	       T26 = T1K - T1R;
+	       T20 = T1W + T1Z;
+	       T24 = T20 - T23;
+	       T28 = T23 + T20;
+	       {
+		    E T1H, T1T, T25, T27;
+		    T1H = W[0];
+		    T1T = W[1];
+		    rio[WS(vs, 1) + WS(is, 2)] = FMA(T1H, T1S, T1T * T24);
+		    iio[WS(vs, 1) + WS(is, 2)] = FNMS(T1T, T1S, T1H * T24);
+		    T25 = W[6];
+		    T27 = W[7];
+		    rio[WS(vs, 4) + WS(is, 2)] = FMA(T25, T26, T27 * T28);
+		    iio[WS(vs, 4) + WS(is, 2)] = FNMS(T27, T26, T25 * T28);
+	       }
+	  }
+	  {
+	       E T2E, T2S, T2Q, T2U, T2w, T2M;
+	       T2w = T2u + T2v;
+	       T2E = T2w + T2D;
+	       T2S = T2w - T2D;
+	       T2M = T2I + T2L;
+	       T2Q = T2M - T2P;
+	       T2U = T2P + T2M;
+	       {
+		    E T2t, T2F, T2R, T2T;
+		    T2t = W[0];
+		    T2F = W[1];
+		    rio[WS(vs, 1) + WS(is, 3)] = FMA(T2t, T2E, T2F * T2Q);
+		    iio[WS(vs, 1) + WS(is, 3)] = FNMS(T2F, T2E, T2t * T2Q);
+		    T2R = W[6];
+		    T2T = W[7];
+		    rio[WS(vs, 4) + WS(is, 3)] = FMA(T2R, T2S, T2T * T2U);
+		    iio[WS(vs, 4) + WS(is, 3)] = FNMS(T2T, T2S, T2R * T2U);
+	       }
+	  }
+	  {
+	       E T2Y, T34, T32, T36, T2W, T31;
+	       T2W = T2v - T2u;
+	       T2Y = T2W - T2X;
+	       T34 = T2W + T2X;
+	       T31 = T2L - T2I;
+	       T32 = T30 + T31;
+	       T36 = T31 - T30;
+	       {
+		    E T2V, T2Z, T33, T35;
+		    T2V = W[2];
+		    T2Z = W[3];
+		    rio[WS(vs, 2) + WS(is, 3)] = FMA(T2V, T2Y, T2Z * T32);
+		    iio[WS(vs, 2) + WS(is, 3)] = FNMS(T2Z, T2Y, T2V * T32);
+		    T33 = W[4];
+		    T35 = W[5];
+		    rio[WS(vs, 3) + WS(is, 3)] = FMA(T33, T34, T35 * T36);
+		    iio[WS(vs, 3) + WS(is, 3)] = FNMS(T35, T34, T33 * T36);
+	       }
+	  }
+	  {
+	       E T3q, T3E, T3C, T3G, T3i, T3y;
+	       T3i = T3g + T3h;
+	       T3q = T3i + T3p;
+	       T3E = T3i - T3p;
+	       T3y = T3u + T3x;
+	       T3C = T3y - T3B;
+	       T3G = T3B + T3y;
+	       {
+		    E T3f, T3r, T3D, T3F;
+		    T3f = W[0];
+		    T3r = W[1];
+		    rio[WS(vs, 1) + WS(is, 4)] = FMA(T3f, T3q, T3r * T3C);
+		    iio[WS(vs, 1) + WS(is, 4)] = FNMS(T3r, T3q, T3f * T3C);
+		    T3D = W[6];
+		    T3F = W[7];
+		    rio[WS(vs, 4) + WS(is, 4)] = FMA(T3D, T3E, T3F * T3G);
+		    iio[WS(vs, 4) + WS(is, 4)] = FNMS(T3F, T3E, T3D * T3G);
+	       }
+	  }
+	  {
+	       E T1q, T1w, T1u, T1y, T1o, T1t;
+	       T1o = TX - TW;
+	       T1q = T1o - T1p;
+	       T1w = T1o + T1p;
+	       T1t = T1d - T1a;
+	       T1u = T1s + T1t;
+	       T1y = T1t - T1s;
+	       {
+		    E T1n, T1r, T1v, T1x;
+		    T1n = W[2];
+		    T1r = W[3];
+		    rio[WS(vs, 2) + WS(is, 1)] = FMA(T1n, T1q, T1r * T1u);
+		    iio[WS(vs, 2) + WS(is, 1)] = FNMS(T1r, T1q, T1n * T1u);
+		    T1v = W[4];
+		    T1x = W[5];
+		    rio[WS(vs, 3) + WS(is, 1)] = FMA(T1v, T1w, T1x * T1y);
+		    iio[WS(vs, 3) + WS(is, 1)] = FNMS(T1x, T1w, T1v * T1y);
+	       }
+	  }
+	  {
+	       E T16, T1k, T1i, T1m, TY, T1e;
+	       TY = TW + TX;
+	       T16 = TY + T15;
+	       T1k = TY - T15;
+	       T1e = T1a + T1d;
+	       T1i = T1e - T1h;
+	       T1m = T1h + T1e;
+	       {
+		    E TV, T17, T1j, T1l;
+		    TV = W[0];
+		    T17 = W[1];
+		    rio[WS(vs, 1) + WS(is, 1)] = FMA(TV, T16, T17 * T1i);
+		    iio[WS(vs, 1) + WS(is, 1)] = FNMS(T17, T16, TV * T1i);
+		    T1j = W[6];
+		    T1l = W[7];
+		    rio[WS(vs, 4) + WS(is, 1)] = FMA(T1j, T1k, T1l * T1m);
+		    iio[WS(vs, 4) + WS(is, 1)] = FNMS(T1l, T1k, T1j * T1m);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 5},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 5, "q1_5", twinstr, {130, 70, 70, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_q1_5) (planner *p) {
+     X(kdft_difsq_register) (p, q1_5, &desc);
+}
diff --git a/src/fftw3/dft/codelets/inplace/q1_6.c b/src/fftw3/dft/codelets/inplace/q1_6.c
new file mode 100644
index 0000000..82bdac3
--- /dev/null
+++ b/src/fftw3/dft/codelets/inplace/q1_6.c
@@ -0,0 +1,654 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:39:14 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twidsq -compact -variables 4 -reload-twiddle -dif -n 6 -name q1_6 -include q.h */
+
+/*
+ * This function contains 276 FP additions, 168 FP multiplications,
+ * (or, 192 additions, 84 multiplications, 84 fused multiply/add),
+ * 85 stack variables, and 144 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: q1_6.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_6.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_6.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ */
+
+#include "q.h"
+
+static const R *q1_6(R *rio, R *iio, const R *W, stride is, stride vs, int m, int dist)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m; i > 0; i = i - 1, rio = rio + dist, iio = iio + dist, W = W + 10) {
+	  E T3, Tc, Tt, TM, TX, T16, T1n, T1G, T2h, T2A, T1R, T20, T2L, T2U, T3b;
+	  E T3u, T3F, T3O, T45, T4o, T4Z, T5i, T4z, T4I, Ta, TP, Tf, Tq, Tn, TN;
+	  E Tu, TJ, T14, T1J, T19, T1k, T1h, T1H, T1o, T1D, T2b, T2B, T2i, T2x, T1Y;
+	  E T2D, T23, T2e, T2S, T3x, T2X, T38, T35, T3v, T3c, T3r, T3M, T4r, T3R, T42;
+	  E T3Z, T4p, T46, T4l, T4T, T5j, T50, T5f, T4G, T5l, T4L, T4W;
+	  {
+	       E T1, T2, T1l, T1m;
+	       T1 = rio[0];
+	       T2 = rio[WS(is, 3)];
+	       T3 = T1 + T2;
+	       Tc = T1 - T2;
+	       {
+		    E Tr, Ts, TV, TW;
+		    Tr = iio[0];
+		    Ts = iio[WS(is, 3)];
+		    Tt = Tr - Ts;
+		    TM = Tr + Ts;
+		    TV = rio[WS(vs, 1)];
+		    TW = rio[WS(vs, 1) + WS(is, 3)];
+		    TX = TV + TW;
+		    T16 = TV - TW;
+	       }
+	       T1l = iio[WS(vs, 1)];
+	       T1m = iio[WS(vs, 1) + WS(is, 3)];
+	       T1n = T1l - T1m;
+	       T1G = T1l + T1m;
+	       {
+		    E T2f, T2g, T1P, T1Q;
+		    T2f = iio[WS(vs, 2)];
+		    T2g = iio[WS(vs, 2) + WS(is, 3)];
+		    T2h = T2f - T2g;
+		    T2A = T2f + T2g;
+		    T1P = rio[WS(vs, 2)];
+		    T1Q = rio[WS(vs, 2) + WS(is, 3)];
+		    T1R = T1P + T1Q;
+		    T20 = T1P - T1Q;
+	       }
+	  }
+	  {
+	       E T2J, T2K, T43, T44;
+	       T2J = rio[WS(vs, 3)];
+	       T2K = rio[WS(vs, 3) + WS(is, 3)];
+	       T2L = T2J + T2K;
+	       T2U = T2J - T2K;
+	       {
+		    E T39, T3a, T3D, T3E;
+		    T39 = iio[WS(vs, 3)];
+		    T3a = iio[WS(vs, 3) + WS(is, 3)];
+		    T3b = T39 - T3a;
+		    T3u = T39 + T3a;
+		    T3D = rio[WS(vs, 4)];
+		    T3E = rio[WS(vs, 4) + WS(is, 3)];
+		    T3F = T3D + T3E;
+		    T3O = T3D - T3E;
+	       }
+	       T43 = iio[WS(vs, 4)];
+	       T44 = iio[WS(vs, 4) + WS(is, 3)];
+	       T45 = T43 - T44;
+	       T4o = T43 + T44;
+	       {
+		    E T4X, T4Y, T4x, T4y;
+		    T4X = iio[WS(vs, 5)];
+		    T4Y = iio[WS(vs, 5) + WS(is, 3)];
+		    T4Z = T4X - T4Y;
+		    T5i = T4X + T4Y;
+		    T4x = rio[WS(vs, 5)];
+		    T4y = rio[WS(vs, 5) + WS(is, 3)];
+		    T4z = T4x + T4y;
+		    T4I = T4x - T4y;
+	       }
+	  }
+	  {
+	       E T6, Td, T9, Te;
+	       {
+		    E T4, T5, T7, T8;
+		    T4 = rio[WS(is, 2)];
+		    T5 = rio[WS(is, 5)];
+		    T6 = T4 + T5;
+		    Td = T4 - T5;
+		    T7 = rio[WS(is, 4)];
+		    T8 = rio[WS(is, 1)];
+		    T9 = T7 + T8;
+		    Te = T7 - T8;
+	       }
+	       Ta = T6 + T9;
+	       TP = KP866025403 * (T9 - T6);
+	       Tf = Td + Te;
+	       Tq = KP866025403 * (Te - Td);
+	  }
+	  {
+	       E Tj, TH, Tm, TI;
+	       {
+		    E Th, Ti, Tk, Tl;
+		    Th = iio[WS(is, 2)];
+		    Ti = iio[WS(is, 5)];
+		    Tj = Th - Ti;
+		    TH = Th + Ti;
+		    Tk = iio[WS(is, 4)];
+		    Tl = iio[WS(is, 1)];
+		    Tm = Tk - Tl;
+		    TI = Tk + Tl;
+	       }
+	       Tn = KP866025403 * (Tj - Tm);
+	       TN = TH + TI;
+	       Tu = Tj + Tm;
+	       TJ = KP866025403 * (TH - TI);
+	  }
+	  {
+	       E T10, T17, T13, T18;
+	       {
+		    E TY, TZ, T11, T12;
+		    TY = rio[WS(vs, 1) + WS(is, 2)];
+		    TZ = rio[WS(vs, 1) + WS(is, 5)];
+		    T10 = TY + TZ;
+		    T17 = TY - TZ;
+		    T11 = rio[WS(vs, 1) + WS(is, 4)];
+		    T12 = rio[WS(vs, 1) + WS(is, 1)];
+		    T13 = T11 + T12;
+		    T18 = T11 - T12;
+	       }
+	       T14 = T10 + T13;
+	       T1J = KP866025403 * (T13 - T10);
+	       T19 = T17 + T18;
+	       T1k = KP866025403 * (T18 - T17);
+	  }
+	  {
+	       E T1d, T1B, T1g, T1C;
+	       {
+		    E T1b, T1c, T1e, T1f;
+		    T1b = iio[WS(vs, 1) + WS(is, 2)];
+		    T1c = iio[WS(vs, 1) + WS(is, 5)];
+		    T1d = T1b - T1c;
+		    T1B = T1b + T1c;
+		    T1e = iio[WS(vs, 1) + WS(is, 4)];
+		    T1f = iio[WS(vs, 1) + WS(is, 1)];
+		    T1g = T1e - T1f;
+		    T1C = T1e + T1f;
+	       }
+	       T1h = KP866025403 * (T1d - T1g);
+	       T1H = T1B + T1C;
+	       T1o = T1d + T1g;
+	       T1D = KP866025403 * (T1B - T1C);
+	  }
+	  {
+	       E T27, T2v, T2a, T2w;
+	       {
+		    E T25, T26, T28, T29;
+		    T25 = iio[WS(vs, 2) + WS(is, 2)];
+		    T26 = iio[WS(vs, 2) + WS(is, 5)];
+		    T27 = T25 - T26;
+		    T2v = T25 + T26;
+		    T28 = iio[WS(vs, 2) + WS(is, 4)];
+		    T29 = iio[WS(vs, 2) + WS(is, 1)];
+		    T2a = T28 - T29;
+		    T2w = T28 + T29;
+	       }
+	       T2b = KP866025403 * (T27 - T2a);
+	       T2B = T2v + T2w;
+	       T2i = T27 + T2a;
+	       T2x = KP866025403 * (T2v - T2w);
+	  }
+	  {
+	       E T1U, T21, T1X, T22;
+	       {
+		    E T1S, T1T, T1V, T1W;
+		    T1S = rio[WS(vs, 2) + WS(is, 2)];
+		    T1T = rio[WS(vs, 2) + WS(is, 5)];
+		    T1U = T1S + T1T;
+		    T21 = T1S - T1T;
+		    T1V = rio[WS(vs, 2) + WS(is, 4)];
+		    T1W = rio[WS(vs, 2) + WS(is, 1)];
+		    T1X = T1V + T1W;
+		    T22 = T1V - T1W;
+	       }
+	       T1Y = T1U + T1X;
+	       T2D = KP866025403 * (T1X - T1U);
+	       T23 = T21 + T22;
+	       T2e = KP866025403 * (T22 - T21);
+	  }
+	  {
+	       E T2O, T2V, T2R, T2W;
+	       {
+		    E T2M, T2N, T2P, T2Q;
+		    T2M = rio[WS(vs, 3) + WS(is, 2)];
+		    T2N = rio[WS(vs, 3) + WS(is, 5)];
+		    T2O = T2M + T2N;
+		    T2V = T2M - T2N;
+		    T2P = rio[WS(vs, 3) + WS(is, 4)];
+		    T2Q = rio[WS(vs, 3) + WS(is, 1)];
+		    T2R = T2P + T2Q;
+		    T2W = T2P - T2Q;
+	       }
+	       T2S = T2O + T2R;
+	       T3x = KP866025403 * (T2R - T2O);
+	       T2X = T2V + T2W;
+	       T38 = KP866025403 * (T2W - T2V);
+	  }
+	  {
+	       E T31, T3p, T34, T3q;
+	       {
+		    E T2Z, T30, T32, T33;
+		    T2Z = iio[WS(vs, 3) + WS(is, 2)];
+		    T30 = iio[WS(vs, 3) + WS(is, 5)];
+		    T31 = T2Z - T30;
+		    T3p = T2Z + T30;
+		    T32 = iio[WS(vs, 3) + WS(is, 4)];
+		    T33 = iio[WS(vs, 3) + WS(is, 1)];
+		    T34 = T32 - T33;
+		    T3q = T32 + T33;
+	       }
+	       T35 = KP866025403 * (T31 - T34);
+	       T3v = T3p + T3q;
+	       T3c = T31 + T34;
+	       T3r = KP866025403 * (T3p - T3q);
+	  }
+	  {
+	       E T3I, T3P, T3L, T3Q;
+	       {
+		    E T3G, T3H, T3J, T3K;
+		    T3G = rio[WS(vs, 4) + WS(is, 2)];
+		    T3H = rio[WS(vs, 4) + WS(is, 5)];
+		    T3I = T3G + T3H;
+		    T3P = T3G - T3H;
+		    T3J = rio[WS(vs, 4) + WS(is, 4)];
+		    T3K = rio[WS(vs, 4) + WS(is, 1)];
+		    T3L = T3J + T3K;
+		    T3Q = T3J - T3K;
+	       }
+	       T3M = T3I + T3L;
+	       T4r = KP866025403 * (T3L - T3I);
+	       T3R = T3P + T3Q;
+	       T42 = KP866025403 * (T3Q - T3P);
+	  }
+	  {
+	       E T3V, T4j, T3Y, T4k;
+	       {
+		    E T3T, T3U, T3W, T3X;
+		    T3T = iio[WS(vs, 4) + WS(is, 2)];
+		    T3U = iio[WS(vs, 4) + WS(is, 5)];
+		    T3V = T3T - T3U;
+		    T4j = T3T + T3U;
+		    T3W = iio[WS(vs, 4) + WS(is, 4)];
+		    T3X = iio[WS(vs, 4) + WS(is, 1)];
+		    T3Y = T3W - T3X;
+		    T4k = T3W + T3X;
+	       }
+	       T3Z = KP866025403 * (T3V - T3Y);
+	       T4p = T4j + T4k;
+	       T46 = T3V + T3Y;
+	       T4l = KP866025403 * (T4j - T4k);
+	  }
+	  {
+	       E T4P, T5d, T4S, T5e;
+	       {
+		    E T4N, T4O, T4Q, T4R;
+		    T4N = iio[WS(vs, 5) + WS(is, 2)];
+		    T4O = iio[WS(vs, 5) + WS(is, 5)];
+		    T4P = T4N - T4O;
+		    T5d = T4N + T4O;
+		    T4Q = iio[WS(vs, 5) + WS(is, 4)];
+		    T4R = iio[WS(vs, 5) + WS(is, 1)];
+		    T4S = T4Q - T4R;
+		    T5e = T4Q + T4R;
+	       }
+	       T4T = KP866025403 * (T4P - T4S);
+	       T5j = T5d + T5e;
+	       T50 = T4P + T4S;
+	       T5f = KP866025403 * (T5d - T5e);
+	  }
+	  {
+	       E T4C, T4J, T4F, T4K;
+	       {
+		    E T4A, T4B, T4D, T4E;
+		    T4A = rio[WS(vs, 5) + WS(is, 2)];
+		    T4B = rio[WS(vs, 5) + WS(is, 5)];
+		    T4C = T4A + T4B;
+		    T4J = T4A - T4B;
+		    T4D = rio[WS(vs, 5) + WS(is, 4)];
+		    T4E = rio[WS(vs, 5) + WS(is, 1)];
+		    T4F = T4D + T4E;
+		    T4K = T4D - T4E;
+	       }
+	       T4G = T4C + T4F;
+	       T5l = KP866025403 * (T4F - T4C);
+	       T4L = T4J + T4K;
+	       T4W = KP866025403 * (T4K - T4J);
+	  }
+	  rio[0] = T3 + Ta;
+	  iio[0] = TM + TN;
+	  rio[WS(is, 1)] = TX + T14;
+	  iio[WS(is, 1)] = T1G + T1H;
+	  rio[WS(is, 3)] = T2L + T2S;
+	  rio[WS(is, 2)] = T1R + T1Y;
+	  iio[WS(is, 2)] = T2A + T2B;
+	  iio[WS(is, 3)] = T3u + T3v;
+	  iio[WS(is, 4)] = T4o + T4p;
+	  iio[WS(is, 5)] = T5i + T5j;
+	  rio[WS(is, 5)] = T4z + T4G;
+	  rio[WS(is, 4)] = T3F + T3M;
+	  {
+	       E T1w, T1y, T1v, T1x;
+	       T1w = T16 + T19;
+	       T1y = T1n + T1o;
+	       T1v = W[4];
+	       T1x = W[5];
+	       rio[WS(vs, 3) + WS(is, 1)] = FMA(T1v, T1w, T1x * T1y);
+	       iio[WS(vs, 3) + WS(is, 1)] = FNMS(T1x, T1w, T1v * T1y);
+	  }
+	  {
+	       E T58, T5a, T57, T59;
+	       T58 = T4I + T4L;
+	       T5a = T4Z + T50;
+	       T57 = W[4];
+	       T59 = W[5];
+	       rio[WS(vs, 3) + WS(is, 5)] = FMA(T57, T58, T59 * T5a);
+	       iio[WS(vs, 3) + WS(is, 5)] = FNMS(T59, T58, T57 * T5a);
+	  }
+	  {
+	       E TC, TE, TB, TD;
+	       TC = Tc + Tf;
+	       TE = Tt + Tu;
+	       TB = W[4];
+	       TD = W[5];
+	       rio[WS(vs, 3)] = FMA(TB, TC, TD * TE);
+	       iio[WS(vs, 3)] = FNMS(TD, TC, TB * TE);
+	  }
+	  {
+	       E T4e, T4g, T4d, T4f;
+	       T4e = T3O + T3R;
+	       T4g = T45 + T46;
+	       T4d = W[4];
+	       T4f = W[5];
+	       rio[WS(vs, 3) + WS(is, 4)] = FMA(T4d, T4e, T4f * T4g);
+	       iio[WS(vs, 3) + WS(is, 4)] = FNMS(T4f, T4e, T4d * T4g);
+	  }
+	  {
+	       E T3k, T3m, T3j, T3l;
+	       T3k = T2U + T2X;
+	       T3m = T3b + T3c;
+	       T3j = W[4];
+	       T3l = W[5];
+	       rio[WS(vs, 3) + WS(is, 3)] = FMA(T3j, T3k, T3l * T3m);
+	       iio[WS(vs, 3) + WS(is, 3)] = FNMS(T3l, T3k, T3j * T3m);
+	  }
+	  {
+	       E T2q, T2s, T2p, T2r;
+	       T2q = T20 + T23;
+	       T2s = T2h + T2i;
+	       T2p = W[4];
+	       T2r = W[5];
+	       rio[WS(vs, 3) + WS(is, 2)] = FMA(T2p, T2q, T2r * T2s);
+	       iio[WS(vs, 3) + WS(is, 2)] = FNMS(T2r, T2q, T2p * T2s);
+	  }
+	  {
+	       E T5g, T5o, T5m, T5q, T5c, T5k;
+	       T5c = FNMS(KP500000000, T4G, T4z);
+	       T5g = T5c - T5f;
+	       T5o = T5c + T5f;
+	       T5k = FNMS(KP500000000, T5j, T5i);
+	       T5m = T5k - T5l;
+	       T5q = T5l + T5k;
+	       {
+		    E T5b, T5h, T5n, T5p;
+		    T5b = W[2];
+		    T5h = W[3];
+		    rio[WS(vs, 2) + WS(is, 5)] = FMA(T5b, T5g, T5h * T5m);
+		    iio[WS(vs, 2) + WS(is, 5)] = FNMS(T5h, T5g, T5b * T5m);
+		    T5n = W[6];
+		    T5p = W[7];
+		    rio[WS(vs, 4) + WS(is, 5)] = FMA(T5n, T5o, T5p * T5q);
+		    iio[WS(vs, 4) + WS(is, 5)] = FNMS(T5p, T5o, T5n * T5q);
+	       }
+	  }
+	  {
+	       E To, Ty, Tw, TA, Tg, Tv;
+	       Tg = FNMS(KP500000000, Tf, Tc);
+	       To = Tg + Tn;
+	       Ty = Tg - Tn;
+	       Tv = FNMS(KP500000000, Tu, Tt);
+	       Tw = Tq + Tv;
+	       TA = Tv - Tq;
+	       {
+		    E Tb, Tp, Tx, Tz;
+		    Tb = W[0];
+		    Tp = W[1];
+		    rio[WS(vs, 1)] = FMA(Tb, To, Tp * Tw);
+		    iio[WS(vs, 1)] = FNMS(Tp, To, Tb * Tw);
+		    Tx = W[8];
+		    Tz = W[9];
+		    rio[WS(vs, 5)] = FMA(Tx, Ty, Tz * TA);
+		    iio[WS(vs, 5)] = FNMS(Tz, Ty, Tx * TA);
+	       }
+	  }
+	  {
+	       E T36, T3g, T3e, T3i, T2Y, T3d;
+	       T2Y = FNMS(KP500000000, T2X, T2U);
+	       T36 = T2Y + T35;
+	       T3g = T2Y - T35;
+	       T3d = FNMS(KP500000000, T3c, T3b);
+	       T3e = T38 + T3d;
+	       T3i = T3d - T38;
+	       {
+		    E T2T, T37, T3f, T3h;
+		    T2T = W[0];
+		    T37 = W[1];
+		    rio[WS(vs, 1) + WS(is, 3)] = FMA(T2T, T36, T37 * T3e);
+		    iio[WS(vs, 1) + WS(is, 3)] = FNMS(T37, T36, T2T * T3e);
+		    T3f = W[8];
+		    T3h = W[9];
+		    rio[WS(vs, 5) + WS(is, 3)] = FMA(T3f, T3g, T3h * T3i);
+		    iio[WS(vs, 5) + WS(is, 3)] = FNMS(T3h, T3g, T3f * T3i);
+	       }
+	  }
+	  {
+	       E T2y, T2G, T2E, T2I, T2u, T2C;
+	       T2u = FNMS(KP500000000, T1Y, T1R);
+	       T2y = T2u - T2x;
+	       T2G = T2u + T2x;
+	       T2C = FNMS(KP500000000, T2B, T2A);
+	       T2E = T2C - T2D;
+	       T2I = T2D + T2C;
+	       {
+		    E T2t, T2z, T2F, T2H;
+		    T2t = W[2];
+		    T2z = W[3];
+		    rio[WS(vs, 2) + WS(is, 2)] = FMA(T2t, T2y, T2z * T2E);
+		    iio[WS(vs, 2) + WS(is, 2)] = FNMS(T2z, T2y, T2t * T2E);
+		    T2F = W[6];
+		    T2H = W[7];
+		    rio[WS(vs, 4) + WS(is, 2)] = FMA(T2F, T2G, T2H * T2I);
+		    iio[WS(vs, 4) + WS(is, 2)] = FNMS(T2H, T2G, T2F * T2I);
+	       }
+	  }
+	  {
+	       E T3s, T3A, T3y, T3C, T3o, T3w;
+	       T3o = FNMS(KP500000000, T2S, T2L);
+	       T3s = T3o - T3r;
+	       T3A = T3o + T3r;
+	       T3w = FNMS(KP500000000, T3v, T3u);
+	       T3y = T3w - T3x;
+	       T3C = T3x + T3w;
+	       {
+		    E T3n, T3t, T3z, T3B;
+		    T3n = W[2];
+		    T3t = W[3];
+		    rio[WS(vs, 2) + WS(is, 3)] = FMA(T3n, T3s, T3t * T3y);
+		    iio[WS(vs, 2) + WS(is, 3)] = FNMS(T3t, T3s, T3n * T3y);
+		    T3z = W[6];
+		    T3B = W[7];
+		    rio[WS(vs, 4) + WS(is, 3)] = FMA(T3z, T3A, T3B * T3C);
+		    iio[WS(vs, 4) + WS(is, 3)] = FNMS(T3B, T3A, T3z * T3C);
+	       }
+	  }
+	  {
+	       E T1E, T1M, T1K, T1O, T1A, T1I;
+	       T1A = FNMS(KP500000000, T14, TX);
+	       T1E = T1A - T1D;
+	       T1M = T1A + T1D;
+	       T1I = FNMS(KP500000000, T1H, T1G);
+	       T1K = T1I - T1J;
+	       T1O = T1J + T1I;
+	       {
+		    E T1z, T1F, T1L, T1N;
+		    T1z = W[2];
+		    T1F = W[3];
+		    rio[WS(vs, 2) + WS(is, 1)] = FMA(T1z, T1E, T1F * T1K);
+		    iio[WS(vs, 2) + WS(is, 1)] = FNMS(T1F, T1E, T1z * T1K);
+		    T1L = W[6];
+		    T1N = W[7];
+		    rio[WS(vs, 4) + WS(is, 1)] = FMA(T1L, T1M, T1N * T1O);
+		    iio[WS(vs, 4) + WS(is, 1)] = FNMS(T1N, T1M, T1L * T1O);
+	       }
+	  }
+	  {
+	       E T4m, T4u, T4s, T4w, T4i, T4q;
+	       T4i = FNMS(KP500000000, T3M, T3F);
+	       T4m = T4i - T4l;
+	       T4u = T4i + T4l;
+	       T4q = FNMS(KP500000000, T4p, T4o);
+	       T4s = T4q - T4r;
+	       T4w = T4r + T4q;
+	       {
+		    E T4h, T4n, T4t, T4v;
+		    T4h = W[2];
+		    T4n = W[3];
+		    rio[WS(vs, 2) + WS(is, 4)] = FMA(T4h, T4m, T4n * T4s);
+		    iio[WS(vs, 2) + WS(is, 4)] = FNMS(T4n, T4m, T4h * T4s);
+		    T4t = W[6];
+		    T4v = W[7];
+		    rio[WS(vs, 4) + WS(is, 4)] = FMA(T4t, T4u, T4v * T4w);
+		    iio[WS(vs, 4) + WS(is, 4)] = FNMS(T4v, T4u, T4t * T4w);
+	       }
+	  }
+	  {
+	       E TK, TS, TQ, TU, TG, TO;
+	       TG = FNMS(KP500000000, Ta, T3);
+	       TK = TG - TJ;
+	       TS = TG + TJ;
+	       TO = FNMS(KP500000000, TN, TM);
+	       TQ = TO - TP;
+	       TU = TP + TO;
+	       {
+		    E TF, TL, TR, TT;
+		    TF = W[2];
+		    TL = W[3];
+		    rio[WS(vs, 2)] = FMA(TF, TK, TL * TQ);
+		    iio[WS(vs, 2)] = FNMS(TL, TK, TF * TQ);
+		    TR = W[6];
+		    TT = W[7];
+		    rio[WS(vs, 4)] = FMA(TR, TS, TT * TU);
+		    iio[WS(vs, 4)] = FNMS(TT, TS, TR * TU);
+	       }
+	  }
+	  {
+	       E T2c, T2m, T2k, T2o, T24, T2j;
+	       T24 = FNMS(KP500000000, T23, T20);
+	       T2c = T24 + T2b;
+	       T2m = T24 - T2b;
+	       T2j = FNMS(KP500000000, T2i, T2h);
+	       T2k = T2e + T2j;
+	       T2o = T2j - T2e;
+	       {
+		    E T1Z, T2d, T2l, T2n;
+		    T1Z = W[0];
+		    T2d = W[1];
+		    rio[WS(vs, 1) + WS(is, 2)] = FMA(T1Z, T2c, T2d * T2k);
+		    iio[WS(vs, 1) + WS(is, 2)] = FNMS(T2d, T2c, T1Z * T2k);
+		    T2l = W[8];
+		    T2n = W[9];
+		    rio[WS(vs, 5) + WS(is, 2)] = FMA(T2l, T2m, T2n * T2o);
+		    iio[WS(vs, 5) + WS(is, 2)] = FNMS(T2n, T2m, T2l * T2o);
+	       }
+	  }
+	  {
+	       E T40, T4a, T48, T4c, T3S, T47;
+	       T3S = FNMS(KP500000000, T3R, T3O);
+	       T40 = T3S + T3Z;
+	       T4a = T3S - T3Z;
+	       T47 = FNMS(KP500000000, T46, T45);
+	       T48 = T42 + T47;
+	       T4c = T47 - T42;
+	       {
+		    E T3N, T41, T49, T4b;
+		    T3N = W[0];
+		    T41 = W[1];
+		    rio[WS(vs, 1) + WS(is, 4)] = FMA(T3N, T40, T41 * T48);
+		    iio[WS(vs, 1) + WS(is, 4)] = FNMS(T41, T40, T3N * T48);
+		    T49 = W[8];
+		    T4b = W[9];
+		    rio[WS(vs, 5) + WS(is, 4)] = FMA(T49, T4a, T4b * T4c);
+		    iio[WS(vs, 5) + WS(is, 4)] = FNMS(T4b, T4a, T49 * T4c);
+	       }
+	  }
+	  {
+	       E T1i, T1s, T1q, T1u, T1a, T1p;
+	       T1a = FNMS(KP500000000, T19, T16);
+	       T1i = T1a + T1h;
+	       T1s = T1a - T1h;
+	       T1p = FNMS(KP500000000, T1o, T1n);
+	       T1q = T1k + T1p;
+	       T1u = T1p - T1k;
+	       {
+		    E T15, T1j, T1r, T1t;
+		    T15 = W[0];
+		    T1j = W[1];
+		    rio[WS(vs, 1) + WS(is, 1)] = FMA(T15, T1i, T1j * T1q);
+		    iio[WS(vs, 1) + WS(is, 1)] = FNMS(T1j, T1i, T15 * T1q);
+		    T1r = W[8];
+		    T1t = W[9];
+		    rio[WS(vs, 5) + WS(is, 1)] = FMA(T1r, T1s, T1t * T1u);
+		    iio[WS(vs, 5) + WS(is, 1)] = FNMS(T1t, T1s, T1r * T1u);
+	       }
+	  }
+	  {
+	       E T4U, T54, T52, T56, T4M, T51;
+	       T4M = FNMS(KP500000000, T4L, T4I);
+	       T4U = T4M + T4T;
+	       T54 = T4M - T4T;
+	       T51 = FNMS(KP500000000, T50, T4Z);
+	       T52 = T4W + T51;
+	       T56 = T51 - T4W;
+	       {
+		    E T4H, T4V, T53, T55;
+		    T4H = W[0];
+		    T4V = W[1];
+		    rio[WS(vs, 1) + WS(is, 5)] = FMA(T4H, T4U, T4V * T52);
+		    iio[WS(vs, 1) + WS(is, 5)] = FNMS(T4V, T4U, T4H * T52);
+		    T53 = W[8];
+		    T55 = W[9];
+		    rio[WS(vs, 5) + WS(is, 5)] = FMA(T53, T54, T55 * T56);
+		    iio[WS(vs, 5) + WS(is, 5)] = FNMS(T55, T54, T53 * T56);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 6},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 6, "q1_6", twinstr, {192, 84, 84, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_q1_6) (planner *p) {
+     X(kdft_difsq_register) (p, q1_6, &desc);
+}
diff --git a/src/fftw3/dft/codelets/inplace/q1_8.c b/src/fftw3/dft/codelets/inplace/q1_8.c
new file mode 100644
index 0000000..84409a7
--- /dev/null
+++ b/src/fftw3/dft/codelets/inplace/q1_8.c
@@ -0,0 +1,1149 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:39:14 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twidsq -compact -variables 4 -reload-twiddle -dif -n 8 -name q1_8 -include q.h */
+
+/*
+ * This function contains 528 FP additions, 256 FP multiplications,
+ * (or, 416 additions, 144 multiplications, 112 fused multiply/add),
+ * 142 stack variables, and 256 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: q1_8.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_8.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ * $Id: q1_8.c,v 1.1 2008/10/17 06:11:08 scuri Exp $
+ */
+
+#include "q.h"
+
+static const R *q1_8(R *rio, R *iio, const R *W, stride is, stride vs, int m, int dist)
+{
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m; i > 0; i = i - 1, rio = rio + dist, iio = iio + dist, W = W + 14) {
+	  E T7, T14, T1g, Tk, TC, TQ, T10, TM, T1w, T2p, T2z, T1H, T1M, T1W, T2j;
+	  E T1V, T7R, T8O, T90, T84, T8m, T8A, T8K, T8w, T9g, Ta9, Taj, T9r, T9w, T9G;
+	  E Ta3, T9F, Te, T17, T1h, Tp, Tu, TE, T11, TD, T1p, T2m, T2y, T1C, T1U;
+	  E T28, T2i, T24, T7Y, T8R, T91, T89, T8e, T8o, T8L, T8n, T99, Ta6, Tai, T9m;
+	  E T9E, T9S, Ta2, T9O, T2H, T3E, T3Q, T2U, T3c, T3q, T3A, T3m, T46, T4Z, T59;
+	  E T4h, T4m, T4w, T4T, T4v, T5h, T6e, T6q, T5u, T5M, T60, T6a, T5W, T6G, T7z;
+	  E T7J, T6R, T6W, T76, T7t, T75, T2O, T3H, T3R, T2Z, T34, T3e, T3B, T3d, T3Z;
+	  E T4W, T58, T4c, T4u, T4I, T4S, T4E, T5o, T6h, T6r, T5z, T5E, T5O, T6b, T5N;
+	  E T6z, T7w, T7I, T6M, T74, T7i, T7s, T7e;
+	  {
+	       E T3, Ty, Tj, TY, T6, Tg, TB, TZ;
+	       {
+		    E T1, T2, Th, Ti;
+		    T1 = rio[0];
+		    T2 = rio[WS(is, 4)];
+		    T3 = T1 + T2;
+		    Ty = T1 - T2;
+		    Th = iio[0];
+		    Ti = iio[WS(is, 4)];
+		    Tj = Th - Ti;
+		    TY = Th + Ti;
+	       }
+	       {
+		    E T4, T5, Tz, TA;
+		    T4 = rio[WS(is, 2)];
+		    T5 = rio[WS(is, 6)];
+		    T6 = T4 + T5;
+		    Tg = T4 - T5;
+		    Tz = iio[WS(is, 2)];
+		    TA = iio[WS(is, 6)];
+		    TB = Tz - TA;
+		    TZ = Tz + TA;
+	       }
+	       T7 = T3 + T6;
+	       T14 = T3 - T6;
+	       T1g = TY + TZ;
+	       Tk = Tg + Tj;
+	       TC = Ty - TB;
+	       TQ = Tj - Tg;
+	       T10 = TY - TZ;
+	       TM = Ty + TB;
+	  }
+	  {
+	       E T1s, T1I, T1L, T2n, T1v, T1D, T1G, T2o;
+	       {
+		    E T1q, T1r, T1J, T1K;
+		    T1q = rio[WS(vs, 1) + WS(is, 1)];
+		    T1r = rio[WS(vs, 1) + WS(is, 5)];
+		    T1s = T1q + T1r;
+		    T1I = T1q - T1r;
+		    T1J = iio[WS(vs, 1) + WS(is, 1)];
+		    T1K = iio[WS(vs, 1) + WS(is, 5)];
+		    T1L = T1J - T1K;
+		    T2n = T1J + T1K;
+	       }
+	       {
+		    E T1t, T1u, T1E, T1F;
+		    T1t = rio[WS(vs, 1) + WS(is, 7)];
+		    T1u = rio[WS(vs, 1) + WS(is, 3)];
+		    T1v = T1t + T1u;
+		    T1D = T1t - T1u;
+		    T1E = iio[WS(vs, 1) + WS(is, 7)];
+		    T1F = iio[WS(vs, 1) + WS(is, 3)];
+		    T1G = T1E - T1F;
+		    T2o = T1E + T1F;
+	       }
+	       T1w = T1s + T1v;
+	       T2p = T2n - T2o;
+	       T2z = T2n + T2o;
+	       T1H = T1D - T1G;
+	       T1M = T1I + T1L;
+	       T1W = T1D + T1G;
+	       T2j = T1v - T1s;
+	       T1V = T1L - T1I;
+	  }
+	  {
+	       E T7N, T8i, T83, T8I, T7Q, T80, T8l, T8J;
+	       {
+		    E T7L, T7M, T81, T82;
+		    T7L = rio[WS(vs, 6)];
+		    T7M = rio[WS(vs, 6) + WS(is, 4)];
+		    T7N = T7L + T7M;
+		    T8i = T7L - T7M;
+		    T81 = iio[WS(vs, 6)];
+		    T82 = iio[WS(vs, 6) + WS(is, 4)];
+		    T83 = T81 - T82;
+		    T8I = T81 + T82;
+	       }
+	       {
+		    E T7O, T7P, T8j, T8k;
+		    T7O = rio[WS(vs, 6) + WS(is, 2)];
+		    T7P = rio[WS(vs, 6) + WS(is, 6)];
+		    T7Q = T7O + T7P;
+		    T80 = T7O - T7P;
+		    T8j = iio[WS(vs, 6) + WS(is, 2)];
+		    T8k = iio[WS(vs, 6) + WS(is, 6)];
+		    T8l = T8j - T8k;
+		    T8J = T8j + T8k;
+	       }
+	       T7R = T7N + T7Q;
+	       T8O = T7N - T7Q;
+	       T90 = T8I + T8J;
+	       T84 = T80 + T83;
+	       T8m = T8i - T8l;
+	       T8A = T83 - T80;
+	       T8K = T8I - T8J;
+	       T8w = T8i + T8l;
+	  }
+	  {
+	       E T9c, T9s, T9v, Ta7, T9f, T9n, T9q, Ta8;
+	       {
+		    E T9a, T9b, T9t, T9u;
+		    T9a = rio[WS(vs, 7) + WS(is, 1)];
+		    T9b = rio[WS(vs, 7) + WS(is, 5)];
+		    T9c = T9a + T9b;
+		    T9s = T9a - T9b;
+		    T9t = iio[WS(vs, 7) + WS(is, 1)];
+		    T9u = iio[WS(vs, 7) + WS(is, 5)];
+		    T9v = T9t - T9u;
+		    Ta7 = T9t + T9u;
+	       }
+	       {
+		    E T9d, T9e, T9o, T9p;
+		    T9d = rio[WS(vs, 7) + WS(is, 7)];
+		    T9e = rio[WS(vs, 7) + WS(is, 3)];
+		    T9f = T9d + T9e;
+		    T9n = T9d - T9e;
+		    T9o = iio[WS(vs, 7) + WS(is, 7)];
+		    T9p = iio[WS(vs, 7) + WS(is, 3)];
+		    T9q = T9o - T9p;
+		    Ta8 = T9o + T9p;
+	       }
+	       T9g = T9c + T9f;
+	       Ta9 = Ta7 - Ta8;
+	       Taj = Ta7 + Ta8;
+	       T9r = T9n - T9q;
+	       T9w = T9s + T9v;
+	       T9G = T9n + T9q;
+	       Ta3 = T9f - T9c;
+	       T9F = T9v - T9s;
+	  }
+	  {
+	       E Ta, Tq, Tt, T15, Td, Tl, To, T16;
+	       {
+		    E T8, T9, Tr, Ts;
+		    T8 = rio[WS(is, 1)];
+		    T9 = rio[WS(is, 5)];
+		    Ta = T8 + T9;
+		    Tq = T8 - T9;
+		    Tr = iio[WS(is, 1)];
+		    Ts = iio[WS(is, 5)];
+		    Tt = Tr - Ts;
+		    T15 = Tr + Ts;
+	       }
+	       {
+		    E Tb, Tc, Tm, Tn;
+		    Tb = rio[WS(is, 7)];
+		    Tc = rio[WS(is, 3)];
+		    Td = Tb + Tc;
+		    Tl = Tb - Tc;
+		    Tm = iio[WS(is, 7)];
+		    Tn = iio[WS(is, 3)];
+		    To = Tm - Tn;
+		    T16 = Tm + Tn;
+	       }
+	       Te = Ta + Td;
+	       T17 = T15 - T16;
+	       T1h = T15 + T16;
+	       Tp = Tl - To;
+	       Tu = Tq + Tt;
+	       TE = Tl + To;
+	       T11 = Td - Ta;
+	       TD = Tt - Tq;
+	  }
+	  {
+	       E T1l, T1Q, T1B, T2g, T1o, T1y, T1T, T2h;
+	       {
+		    E T1j, T1k, T1z, T1A;
+		    T1j = rio[WS(vs, 1)];
+		    T1k = rio[WS(vs, 1) + WS(is, 4)];
+		    T1l = T1j + T1k;
+		    T1Q = T1j - T1k;
+		    T1z = iio[WS(vs, 1)];
+		    T1A = iio[WS(vs, 1) + WS(is, 4)];
+		    T1B = T1z - T1A;
+		    T2g = T1z + T1A;
+	       }
+	       {
+		    E T1m, T1n, T1R, T1S;
+		    T1m = rio[WS(vs, 1) + WS(is, 2)];
+		    T1n = rio[WS(vs, 1) + WS(is, 6)];
+		    T1o = T1m + T1n;
+		    T1y = T1m - T1n;
+		    T1R = iio[WS(vs, 1) + WS(is, 2)];
+		    T1S = iio[WS(vs, 1) + WS(is, 6)];
+		    T1T = T1R - T1S;
+		    T2h = T1R + T1S;
+	       }
+	       T1p = T1l + T1o;
+	       T2m = T1l - T1o;
+	       T2y = T2g + T2h;
+	       T1C = T1y + T1B;
+	       T1U = T1Q - T1T;
+	       T28 = T1B - T1y;
+	       T2i = T2g - T2h;
+	       T24 = T1Q + T1T;
+	  }
+	  {
+	       E T7U, T8a, T8d, T8P, T7X, T85, T88, T8Q;
+	       {
+		    E T7S, T7T, T8b, T8c;
+		    T7S = rio[WS(vs, 6) + WS(is, 1)];
+		    T7T = rio[WS(vs, 6) + WS(is, 5)];
+		    T7U = T7S + T7T;
+		    T8a = T7S - T7T;
+		    T8b = iio[WS(vs, 6) + WS(is, 1)];
+		    T8c = iio[WS(vs, 6) + WS(is, 5)];
+		    T8d = T8b - T8c;
+		    T8P = T8b + T8c;
+	       }
+	       {
+		    E T7V, T7W, T86, T87;
+		    T7V = rio[WS(vs, 6) + WS(is, 7)];
+		    T7W = rio[WS(vs, 6) + WS(is, 3)];
+		    T7X = T7V + T7W;
+		    T85 = T7V - T7W;
+		    T86 = iio[WS(vs, 6) + WS(is, 7)];
+		    T87 = iio[WS(vs, 6) + WS(is, 3)];
+		    T88 = T86 - T87;
+		    T8Q = T86 + T87;
+	       }
+	       T7Y = T7U + T7X;
+	       T8R = T8P - T8Q;
+	       T91 = T8P + T8Q;
+	       T89 = T85 - T88;
+	       T8e = T8a + T8d;
+	       T8o = T85 + T88;
+	       T8L = T7X - T7U;
+	       T8n = T8d - T8a;
+	  }
+	  {
+	       E T95, T9A, T9l, Ta0, T98, T9i, T9D, Ta1;
+	       {
+		    E T93, T94, T9j, T9k;
+		    T93 = rio[WS(vs, 7)];
+		    T94 = rio[WS(vs, 7) + WS(is, 4)];
+		    T95 = T93 + T94;
+		    T9A = T93 - T94;
+		    T9j = iio[WS(vs, 7)];
+		    T9k = iio[WS(vs, 7) + WS(is, 4)];
+		    T9l = T9j - T9k;
+		    Ta0 = T9j + T9k;
+	       }
+	       {
+		    E T96, T97, T9B, T9C;
+		    T96 = rio[WS(vs, 7) + WS(is, 2)];
+		    T97 = rio[WS(vs, 7) + WS(is, 6)];
+		    T98 = T96 + T97;
+		    T9i = T96 - T97;
+		    T9B = iio[WS(vs, 7) + WS(is, 2)];
+		    T9C = iio[WS(vs, 7) + WS(is, 6)];
+		    T9D = T9B - T9C;
+		    Ta1 = T9B + T9C;
+	       }
+	       T99 = T95 + T98;
+	       Ta6 = T95 - T98;
+	       Tai = Ta0 + Ta1;
+	       T9m = T9i + T9l;
+	       T9E = T9A - T9D;
+	       T9S = T9l - T9i;
+	       Ta2 = Ta0 - Ta1;
+	       T9O = T9A + T9D;
+	  }
+	  {
+	       E T2D, T38, T2T, T3y, T2G, T2Q, T3b, T3z;
+	       {
+		    E T2B, T2C, T2R, T2S;
+		    T2B = rio[WS(vs, 2)];
+		    T2C = rio[WS(vs, 2) + WS(is, 4)];
+		    T2D = T2B + T2C;
+		    T38 = T2B - T2C;
+		    T2R = iio[WS(vs, 2)];
+		    T2S = iio[WS(vs, 2) + WS(is, 4)];
+		    T2T = T2R - T2S;
+		    T3y = T2R + T2S;
+	       }
+	       {
+		    E T2E, T2F, T39, T3a;
+		    T2E = rio[WS(vs, 2) + WS(is, 2)];
+		    T2F = rio[WS(vs, 2) + WS(is, 6)];
+		    T2G = T2E + T2F;
+		    T2Q = T2E - T2F;
+		    T39 = iio[WS(vs, 2) + WS(is, 2)];
+		    T3a = iio[WS(vs, 2) + WS(is, 6)];
+		    T3b = T39 - T3a;
+		    T3z = T39 + T3a;
+	       }
+	       T2H = T2D + T2G;
+	       T3E = T2D - T2G;
+	       T3Q = T3y + T3z;
+	       T2U = T2Q + T2T;
+	       T3c = T38 - T3b;
+	       T3q = T2T - T2Q;
+	       T3A = T3y - T3z;
+	       T3m = T38 + T3b;
+	  }
+	  {
+	       E T42, T4i, T4l, T4X, T45, T4d, T4g, T4Y;
+	       {
+		    E T40, T41, T4j, T4k;
+		    T40 = rio[WS(vs, 3) + WS(is, 1)];
+		    T41 = rio[WS(vs, 3) + WS(is, 5)];
+		    T42 = T40 + T41;
+		    T4i = T40 - T41;
+		    T4j = iio[WS(vs, 3) + WS(is, 1)];
+		    T4k = iio[WS(vs, 3) + WS(is, 5)];
+		    T4l = T4j - T4k;
+		    T4X = T4j + T4k;
+	       }
+	       {
+		    E T43, T44, T4e, T4f;
+		    T43 = rio[WS(vs, 3) + WS(is, 7)];
+		    T44 = rio[WS(vs, 3) + WS(is, 3)];
+		    T45 = T43 + T44;
+		    T4d = T43 - T44;
+		    T4e = iio[WS(vs, 3) + WS(is, 7)];
+		    T4f = iio[WS(vs, 3) + WS(is, 3)];
+		    T4g = T4e - T4f;
+		    T4Y = T4e + T4f;
+	       }
+	       T46 = T42 + T45;
+	       T4Z = T4X - T4Y;
+	       T59 = T4X + T4Y;
+	       T4h = T4d - T4g;
+	       T4m = T4i + T4l;
+	       T4w = T4d + T4g;
+	       T4T = T45 - T42;
+	       T4v = T4l - T4i;
+	  }
+	  {
+	       E T5d, T5I, T5t, T68, T5g, T5q, T5L, T69;
+	       {
+		    E T5b, T5c, T5r, T5s;
+		    T5b = rio[WS(vs, 4)];
+		    T5c = rio[WS(vs, 4) + WS(is, 4)];
+		    T5d = T5b + T5c;
+		    T5I = T5b - T5c;
+		    T5r = iio[WS(vs, 4)];
+		    T5s = iio[WS(vs, 4) + WS(is, 4)];
+		    T5t = T5r - T5s;
+		    T68 = T5r + T5s;
+	       }
+	       {
+		    E T5e, T5f, T5J, T5K;
+		    T5e = rio[WS(vs, 4) + WS(is, 2)];
+		    T5f = rio[WS(vs, 4) + WS(is, 6)];
+		    T5g = T5e + T5f;
+		    T5q = T5e - T5f;
+		    T5J = iio[WS(vs, 4) + WS(is, 2)];
+		    T5K = iio[WS(vs, 4) + WS(is, 6)];
+		    T5L = T5J - T5K;
+		    T69 = T5J + T5K;
+	       }
+	       T5h = T5d + T5g;
+	       T6e = T5d - T5g;
+	       T6q = T68 + T69;
+	       T5u = T5q + T5t;
+	       T5M = T5I - T5L;
+	       T60 = T5t - T5q;
+	       T6a = T68 - T69;
+	       T5W = T5I + T5L;
+	  }
+	  {
+	       E T6C, T6S, T6V, T7x, T6F, T6N, T6Q, T7y;
+	       {
+		    E T6A, T6B, T6T, T6U;
+		    T6A = rio[WS(vs, 5) + WS(is, 1)];
+		    T6B = rio[WS(vs, 5) + WS(is, 5)];
+		    T6C = T6A + T6B;
+		    T6S = T6A - T6B;
+		    T6T = iio[WS(vs, 5) + WS(is, 1)];
+		    T6U = iio[WS(vs, 5) + WS(is, 5)];
+		    T6V = T6T - T6U;
+		    T7x = T6T + T6U;
+	       }
+	       {
+		    E T6D, T6E, T6O, T6P;
+		    T6D = rio[WS(vs, 5) + WS(is, 7)];
+		    T6E = rio[WS(vs, 5) + WS(is, 3)];
+		    T6F = T6D + T6E;
+		    T6N = T6D - T6E;
+		    T6O = iio[WS(vs, 5) + WS(is, 7)];
+		    T6P = iio[WS(vs, 5) + WS(is, 3)];
+		    T6Q = T6O - T6P;
+		    T7y = T6O + T6P;
+	       }
+	       T6G = T6C + T6F;
+	       T7z = T7x - T7y;
+	       T7J = T7x + T7y;
+	       T6R = T6N - T6Q;
+	       T6W = T6S + T6V;
+	       T76 = T6N + T6Q;
+	       T7t = T6F - T6C;
+	       T75 = T6V - T6S;
+	  }
+	  {
+	       E T2K, T30, T33, T3F, T2N, T2V, T2Y, T3G;
+	       {
+		    E T2I, T2J, T31, T32;
+		    T2I = rio[WS(vs, 2) + WS(is, 1)];
+		    T2J = rio[WS(vs, 2) + WS(is, 5)];
+		    T2K = T2I + T2J;
+		    T30 = T2I - T2J;
+		    T31 = iio[WS(vs, 2) + WS(is, 1)];
+		    T32 = iio[WS(vs, 2) + WS(is, 5)];
+		    T33 = T31 - T32;
+		    T3F = T31 + T32;
+	       }
+	       {
+		    E T2L, T2M, T2W, T2X;
+		    T2L = rio[WS(vs, 2) + WS(is, 7)];
+		    T2M = rio[WS(vs, 2) + WS(is, 3)];
+		    T2N = T2L + T2M;
+		    T2V = T2L - T2M;
+		    T2W = iio[WS(vs, 2) + WS(is, 7)];
+		    T2X = iio[WS(vs, 2) + WS(is, 3)];
+		    T2Y = T2W - T2X;
+		    T3G = T2W + T2X;
+	       }
+	       T2O = T2K + T2N;
+	       T3H = T3F - T3G;
+	       T3R = T3F + T3G;
+	       T2Z = T2V - T2Y;
+	       T34 = T30 + T33;
+	       T3e = T2V + T2Y;
+	       T3B = T2N - T2K;
+	       T3d = T33 - T30;
+	  }
+	  {
+	       E T3V, T4q, T4b, T4Q, T3Y, T48, T4t, T4R;
+	       {
+		    E T3T, T3U, T49, T4a;
+		    T3T = rio[WS(vs, 3)];
+		    T3U = rio[WS(vs, 3) + WS(is, 4)];
+		    T3V = T3T + T3U;
+		    T4q = T3T - T3U;
+		    T49 = iio[WS(vs, 3)];
+		    T4a = iio[WS(vs, 3) + WS(is, 4)];
+		    T4b = T49 - T4a;
+		    T4Q = T49 + T4a;
+	       }
+	       {
+		    E T3W, T3X, T4r, T4s;
+		    T3W = rio[WS(vs, 3) + WS(is, 2)];
+		    T3X = rio[WS(vs, 3) + WS(is, 6)];
+		    T3Y = T3W + T3X;
+		    T48 = T3W - T3X;
+		    T4r = iio[WS(vs, 3) + WS(is, 2)];
+		    T4s = iio[WS(vs, 3) + WS(is, 6)];
+		    T4t = T4r - T4s;
+		    T4R = T4r + T4s;
+	       }
+	       T3Z = T3V + T3Y;
+	       T4W = T3V - T3Y;
+	       T58 = T4Q + T4R;
+	       T4c = T48 + T4b;
+	       T4u = T4q - T4t;
+	       T4I = T4b - T48;
+	       T4S = T4Q - T4R;
+	       T4E = T4q + T4t;
+	  }
+	  {
+	       E T5k, T5A, T5D, T6f, T5n, T5v, T5y, T6g;
+	       {
+		    E T5i, T5j, T5B, T5C;
+		    T5i = rio[WS(vs, 4) + WS(is, 1)];
+		    T5j = rio[WS(vs, 4) + WS(is, 5)];
+		    T5k = T5i + T5j;
+		    T5A = T5i - T5j;
+		    T5B = iio[WS(vs, 4) + WS(is, 1)];
+		    T5C = iio[WS(vs, 4) + WS(is, 5)];
+		    T5D = T5B - T5C;
+		    T6f = T5B + T5C;
+	       }
+	       {
+		    E T5l, T5m, T5w, T5x;
+		    T5l = rio[WS(vs, 4) + WS(is, 7)];
+		    T5m = rio[WS(vs, 4) + WS(is, 3)];
+		    T5n = T5l + T5m;
+		    T5v = T5l - T5m;
+		    T5w = iio[WS(vs, 4) + WS(is, 7)];
+		    T5x = iio[WS(vs, 4) + WS(is, 3)];
+		    T5y = T5w - T5x;
+		    T6g = T5w + T5x;
+	       }
+	       T5o = T5k + T5n;
+	       T6h = T6f - T6g;
+	       T6r = T6f + T6g;
+	       T5z = T5v - T5y;
+	       T5E = T5A + T5D;
+	       T5O = T5v + T5y;
+	       T6b = T5n - T5k;
+	       T5N = T5D - T5A;
+	  }
+	  {
+	       E T6v, T70, T6L, T7q, T6y, T6I, T73, T7r;
+	       {
+		    E T6t, T6u, T6J, T6K;
+		    T6t = rio[WS(vs, 5)];
+		    T6u = rio[WS(vs, 5) + WS(is, 4)];
+		    T6v = T6t + T6u;
+		    T70 = T6t - T6u;
+		    T6J = iio[WS(vs, 5)];
+		    T6K = iio[WS(vs, 5) + WS(is, 4)];
+		    T6L = T6J - T6K;
+		    T7q = T6J + T6K;
+	       }
+	       {
+		    E T6w, T6x, T71, T72;
+		    T6w = rio[WS(vs, 5) + WS(is, 2)];
+		    T6x = rio[WS(vs, 5) + WS(is, 6)];
+		    T6y = T6w + T6x;
+		    T6I = T6w - T6x;
+		    T71 = iio[WS(vs, 5) + WS(is, 2)];
+		    T72 = iio[WS(vs, 5) + WS(is, 6)];
+		    T73 = T71 - T72;
+		    T7r = T71 + T72;
+	       }
+	       T6z = T6v + T6y;
+	       T7w = T6v - T6y;
+	       T7I = T7q + T7r;
+	       T6M = T6I + T6L;
+	       T74 = T70 - T73;
+	       T7i = T6L - T6I;
+	       T7s = T7q - T7r;
+	       T7e = T70 + T73;
+	  }
+	  rio[0] = T7 + Te;
+	  iio[0] = T1g + T1h;
+	  rio[WS(is, 1)] = T1p + T1w;
+	  iio[WS(is, 1)] = T2y + T2z;
+	  rio[WS(is, 3)] = T3Z + T46;
+	  rio[WS(is, 2)] = T2H + T2O;
+	  iio[WS(is, 2)] = T3Q + T3R;
+	  iio[WS(is, 3)] = T58 + T59;
+	  rio[WS(is, 6)] = T7R + T7Y;
+	  iio[WS(is, 6)] = T90 + T91;
+	  iio[WS(is, 5)] = T7I + T7J;
+	  rio[WS(is, 5)] = T6z + T6G;
+	  iio[WS(is, 4)] = T6q + T6r;
+	  rio[WS(is, 4)] = T5h + T5o;
+	  rio[WS(is, 7)] = T99 + T9g;
+	  iio[WS(is, 7)] = Tai + Taj;
+	  {
+	       E T12, T18, TX, T13;
+	       T12 = T10 - T11;
+	       T18 = T14 - T17;
+	       TX = W[10];
+	       T13 = W[11];
+	       iio[WS(vs, 6)] = FNMS(T13, T18, TX * T12);
+	       rio[WS(vs, 6)] = FMA(T13, T12, TX * T18);
+	  }
+	  {
+	       E Tag, Tak, Taf, Tah;
+	       Tag = T99 - T9g;
+	       Tak = Tai - Taj;
+	       Taf = W[6];
+	       Tah = W[7];
+	       rio[WS(vs, 4) + WS(is, 7)] = FMA(Taf, Tag, Tah * Tak);
+	       iio[WS(vs, 4) + WS(is, 7)] = FNMS(Tah, Tag, Taf * Tak);
+	  }
+	  {
+	       E T8M, T8S, T8H, T8N;
+	       T8M = T8K - T8L;
+	       T8S = T8O - T8R;
+	       T8H = W[10];
+	       T8N = W[11];
+	       iio[WS(vs, 6) + WS(is, 6)] = FNMS(T8N, T8S, T8H * T8M);
+	       rio[WS(vs, 6) + WS(is, 6)] = FMA(T8N, T8M, T8H * T8S);
+	  }
+	  {
+	       E T2k, T2q, T2f, T2l;
+	       T2k = T2i - T2j;
+	       T2q = T2m - T2p;
+	       T2f = W[10];
+	       T2l = W[11];
+	       iio[WS(vs, 6) + WS(is, 1)] = FNMS(T2l, T2q, T2f * T2k);
+	       rio[WS(vs, 6) + WS(is, 1)] = FMA(T2l, T2k, T2f * T2q);
+	  }
+	  {
+	       E Ta4, Taa, T9Z, Ta5;
+	       Ta4 = Ta2 - Ta3;
+	       Taa = Ta6 - Ta9;
+	       T9Z = W[10];
+	       Ta5 = W[11];
+	       iio[WS(vs, 6) + WS(is, 7)] = FNMS(Ta5, Taa, T9Z * Ta4);
+	       rio[WS(vs, 6) + WS(is, 7)] = FMA(Ta5, Ta4, T9Z * Taa);
+	  }
+	  {
+	       E T8Y, T92, T8X, T8Z;
+	       T8Y = T7R - T7Y;
+	       T92 = T90 - T91;
+	       T8X = W[6];
+	       T8Z = W[7];
+	       rio[WS(vs, 4) + WS(is, 6)] = FMA(T8X, T8Y, T8Z * T92);
+	       iio[WS(vs, 4) + WS(is, 6)] = FNMS(T8Z, T8Y, T8X * T92);
+	  }
+	  {
+	       E T2w, T2A, T2v, T2x;
+	       T2w = T1p - T1w;
+	       T2A = T2y - T2z;
+	       T2v = W[6];
+	       T2x = W[7];
+	       rio[WS(vs, 4) + WS(is, 1)] = FMA(T2v, T2w, T2x * T2A);
+	       iio[WS(vs, 4) + WS(is, 1)] = FNMS(T2x, T2w, T2v * T2A);
+	  }
+	  {
+	       E Tac, Tae, Tab, Tad;
+	       Tac = Ta3 + Ta2;
+	       Tae = Ta6 + Ta9;
+	       Tab = W[2];
+	       Tad = W[3];
+	       iio[WS(vs, 2) + WS(is, 7)] = FNMS(Tad, Tae, Tab * Tac);
+	       rio[WS(vs, 2) + WS(is, 7)] = FMA(Tad, Tac, Tab * Tae);
+	  }
+	  {
+	       E T8U, T8W, T8T, T8V;
+	       T8U = T8L + T8K;
+	       T8W = T8O + T8R;
+	       T8T = W[2];
+	       T8V = W[3];
+	       iio[WS(vs, 2) + WS(is, 6)] = FNMS(T8V, T8W, T8T * T8U);
+	       rio[WS(vs, 2) + WS(is, 6)] = FMA(T8V, T8U, T8T * T8W);
+	  }
+	  {
+	       E T1a, T1c, T19, T1b;
+	       T1a = T11 + T10;
+	       T1c = T14 + T17;
+	       T19 = W[2];
+	       T1b = W[3];
+	       iio[WS(vs, 2)] = FNMS(T1b, T1c, T19 * T1a);
+	       rio[WS(vs, 2)] = FMA(T1b, T1a, T19 * T1c);
+	  }
+	  {
+	       E T1e, T1i, T1d, T1f;
+	       T1e = T7 - Te;
+	       T1i = T1g - T1h;
+	       T1d = W[6];
+	       T1f = W[7];
+	       rio[WS(vs, 4)] = FMA(T1d, T1e, T1f * T1i);
+	       iio[WS(vs, 4)] = FNMS(T1f, T1e, T1d * T1i);
+	  }
+	  {
+	       E T2s, T2u, T2r, T2t;
+	       T2s = T2j + T2i;
+	       T2u = T2m + T2p;
+	       T2r = W[2];
+	       T2t = W[3];
+	       iio[WS(vs, 2) + WS(is, 1)] = FNMS(T2t, T2u, T2r * T2s);
+	       rio[WS(vs, 2) + WS(is, 1)] = FMA(T2t, T2s, T2r * T2u);
+	  }
+	  {
+	       E T3C, T3I, T3x, T3D;
+	       T3C = T3A - T3B;
+	       T3I = T3E - T3H;
+	       T3x = W[10];
+	       T3D = W[11];
+	       iio[WS(vs, 6) + WS(is, 2)] = FNMS(T3D, T3I, T3x * T3C);
+	       rio[WS(vs, 6) + WS(is, 2)] = FMA(T3D, T3C, T3x * T3I);
+	  }
+	  {
+	       E T4U, T50, T4P, T4V;
+	       T4U = T4S - T4T;
+	       T50 = T4W - T4Z;
+	       T4P = W[10];
+	       T4V = W[11];
+	       iio[WS(vs, 6) + WS(is, 3)] = FNMS(T4V, T50, T4P * T4U);
+	       rio[WS(vs, 6) + WS(is, 3)] = FMA(T4V, T4U, T4P * T50);
+	  }
+	  {
+	       E T56, T5a, T55, T57;
+	       T56 = T3Z - T46;
+	       T5a = T58 - T59;
+	       T55 = W[6];
+	       T57 = W[7];
+	       rio[WS(vs, 4) + WS(is, 3)] = FMA(T55, T56, T57 * T5a);
+	       iio[WS(vs, 4) + WS(is, 3)] = FNMS(T57, T56, T55 * T5a);
+	  }
+	  {
+	       E T6o, T6s, T6n, T6p;
+	       T6o = T5h - T5o;
+	       T6s = T6q - T6r;
+	       T6n = W[6];
+	       T6p = W[7];
+	       rio[WS(vs, 4) + WS(is, 4)] = FMA(T6n, T6o, T6p * T6s);
+	       iio[WS(vs, 4) + WS(is, 4)] = FNMS(T6p, T6o, T6n * T6s);
+	  }
+	  {
+	       E T7u, T7A, T7p, T7v;
+	       T7u = T7s - T7t;
+	       T7A = T7w - T7z;
+	       T7p = W[10];
+	       T7v = W[11];
+	       iio[WS(vs, 6) + WS(is, 5)] = FNMS(T7v, T7A, T7p * T7u);
+	       rio[WS(vs, 6) + WS(is, 5)] = FMA(T7v, T7u, T7p * T7A);
+	  }
+	  {
+	       E T6c, T6i, T67, T6d;
+	       T6c = T6a - T6b;
+	       T6i = T6e - T6h;
+	       T67 = W[10];
+	       T6d = W[11];
+	       iio[WS(vs, 6) + WS(is, 4)] = FNMS(T6d, T6i, T67 * T6c);
+	       rio[WS(vs, 6) + WS(is, 4)] = FMA(T6d, T6c, T67 * T6i);
+	  }
+	  {
+	       E T7G, T7K, T7F, T7H;
+	       T7G = T6z - T6G;
+	       T7K = T7I - T7J;
+	       T7F = W[6];
+	       T7H = W[7];
+	       rio[WS(vs, 4) + WS(is, 5)] = FMA(T7F, T7G, T7H * T7K);
+	       iio[WS(vs, 4) + WS(is, 5)] = FNMS(T7H, T7G, T7F * T7K);
+	  }
+	  {
+	       E T3O, T3S, T3N, T3P;
+	       T3O = T2H - T2O;
+	       T3S = T3Q - T3R;
+	       T3N = W[6];
+	       T3P = W[7];
+	       rio[WS(vs, 4) + WS(is, 2)] = FMA(T3N, T3O, T3P * T3S);
+	       iio[WS(vs, 4) + WS(is, 2)] = FNMS(T3P, T3O, T3N * T3S);
+	  }
+	  {
+	       E T3K, T3M, T3J, T3L;
+	       T3K = T3B + T3A;
+	       T3M = T3E + T3H;
+	       T3J = W[2];
+	       T3L = W[3];
+	       iio[WS(vs, 2) + WS(is, 2)] = FNMS(T3L, T3M, T3J * T3K);
+	       rio[WS(vs, 2) + WS(is, 2)] = FMA(T3L, T3K, T3J * T3M);
+	  }
+	  {
+	       E T7C, T7E, T7B, T7D;
+	       T7C = T7t + T7s;
+	       T7E = T7w + T7z;
+	       T7B = W[2];
+	       T7D = W[3];
+	       iio[WS(vs, 2) + WS(is, 5)] = FNMS(T7D, T7E, T7B * T7C);
+	       rio[WS(vs, 2) + WS(is, 5)] = FMA(T7D, T7C, T7B * T7E);
+	  }
+	  {
+	       E T6k, T6m, T6j, T6l;
+	       T6k = T6b + T6a;
+	       T6m = T6e + T6h;
+	       T6j = W[2];
+	       T6l = W[3];
+	       iio[WS(vs, 2) + WS(is, 4)] = FNMS(T6l, T6m, T6j * T6k);
+	       rio[WS(vs, 2) + WS(is, 4)] = FMA(T6l, T6k, T6j * T6m);
+	  }
+	  {
+	       E T52, T54, T51, T53;
+	       T52 = T4T + T4S;
+	       T54 = T4W + T4Z;
+	       T51 = W[2];
+	       T53 = W[3];
+	       iio[WS(vs, 2) + WS(is, 3)] = FNMS(T53, T54, T51 * T52);
+	       rio[WS(vs, 2) + WS(is, 3)] = FMA(T53, T52, T51 * T54);
+	  }
+	  {
+	       E T5G, T5S, T5Q, T5U, T5F, T5P;
+	       T5F = KP707106781 * (T5z - T5E);
+	       T5G = T5u - T5F;
+	       T5S = T5u + T5F;
+	       T5P = KP707106781 * (T5N - T5O);
+	       T5Q = T5M - T5P;
+	       T5U = T5M + T5P;
+	       {
+		    E T5p, T5H, T5R, T5T;
+		    T5p = W[12];
+		    T5H = W[13];
+		    iio[WS(vs, 7) + WS(is, 4)] = FNMS(T5H, T5Q, T5p * T5G);
+		    rio[WS(vs, 7) + WS(is, 4)] = FMA(T5H, T5G, T5p * T5Q);
+		    T5R = W[4];
+		    T5T = W[5];
+		    iio[WS(vs, 3) + WS(is, 4)] = FNMS(T5T, T5U, T5R * T5S);
+		    rio[WS(vs, 3) + WS(is, 4)] = FMA(T5T, T5S, T5R * T5U);
+	       }
+	  }
+	  {
+	       E Tw, TI, TG, TK, Tv, TF;
+	       Tv = KP707106781 * (Tp - Tu);
+	       Tw = Tk - Tv;
+	       TI = Tk + Tv;
+	       TF = KP707106781 * (TD - TE);
+	       TG = TC - TF;
+	       TK = TC + TF;
+	       {
+		    E Tf, Tx, TH, TJ;
+		    Tf = W[12];
+		    Tx = W[13];
+		    iio[WS(vs, 7)] = FNMS(Tx, TG, Tf * Tw);
+		    rio[WS(vs, 7)] = FMA(Tx, Tw, Tf * TG);
+		    TH = W[4];
+		    TJ = W[5];
+		    iio[WS(vs, 3)] = FNMS(TJ, TK, TH * TI);
+		    rio[WS(vs, 3)] = FMA(TJ, TI, TH * TK);
+	       }
+	  }
+	  {
+	       E T9Q, T9W, T9U, T9Y, T9P, T9T;
+	       T9P = KP707106781 * (T9w + T9r);
+	       T9Q = T9O - T9P;
+	       T9W = T9O + T9P;
+	       T9T = KP707106781 * (T9F + T9G);
+	       T9U = T9S - T9T;
+	       T9Y = T9S + T9T;
+	       {
+		    E T9N, T9R, T9V, T9X;
+		    T9N = W[8];
+		    T9R = W[9];
+		    rio[WS(vs, 5) + WS(is, 7)] = FMA(T9N, T9Q, T9R * T9U);
+		    iio[WS(vs, 5) + WS(is, 7)] = FNMS(T9R, T9Q, T9N * T9U);
+		    T9V = W[0];
+		    T9X = W[1];
+		    rio[WS(vs, 1) + WS(is, 7)] = FMA(T9V, T9W, T9X * T9Y);
+		    iio[WS(vs, 1) + WS(is, 7)] = FNMS(T9X, T9W, T9V * T9Y);
+	       }
+	  }
+	  {
+	       E T36, T3i, T3g, T3k, T35, T3f;
+	       T35 = KP707106781 * (T2Z - T34);
+	       T36 = T2U - T35;
+	       T3i = T2U + T35;
+	       T3f = KP707106781 * (T3d - T3e);
+	       T3g = T3c - T3f;
+	       T3k = T3c + T3f;
+	       {
+		    E T2P, T37, T3h, T3j;
+		    T2P = W[12];
+		    T37 = W[13];
+		    iio[WS(vs, 7) + WS(is, 2)] = FNMS(T37, T3g, T2P * T36);
+		    rio[WS(vs, 7) + WS(is, 2)] = FMA(T37, T36, T2P * T3g);
+		    T3h = W[4];
+		    T3j = W[5];
+		    iio[WS(vs, 3) + WS(is, 2)] = FNMS(T3j, T3k, T3h * T3i);
+		    rio[WS(vs, 3) + WS(is, 2)] = FMA(T3j, T3i, T3h * T3k);
+	       }
+	  }
+	  {
+	       E T5Y, T64, T62, T66, T5X, T61;
+	       T5X = KP707106781 * (T5E + T5z);
+	       T5Y = T5W - T5X;
+	       T64 = T5W + T5X;
+	       T61 = KP707106781 * (T5N + T5O);
+	       T62 = T60 - T61;
+	       T66 = T60 + T61;
+	       {
+		    E T5V, T5Z, T63, T65;
+		    T5V = W[8];
+		    T5Z = W[9];
+		    rio[WS(vs, 5) + WS(is, 4)] = FMA(T5V, T5Y, T5Z * T62);
+		    iio[WS(vs, 5) + WS(is, 4)] = FNMS(T5Z, T5Y, T5V * T62);
+		    T63 = W[0];
+		    T65 = W[1];
+		    rio[WS(vs, 1) + WS(is, 4)] = FMA(T63, T64, T65 * T66);
+		    iio[WS(vs, 1) + WS(is, 4)] = FNMS(T65, T64, T63 * T66);
+	       }
+	  }
+	  {
+	       E T7g, T7m, T7k, T7o, T7f, T7j;
+	       T7f = KP707106781 * (T6W + T6R);
+	       T7g = T7e - T7f;
+	       T7m = T7e + T7f;
+	       T7j = KP707106781 * (T75 + T76);
+	       T7k = T7i - T7j;
+	       T7o = T7i + T7j;
+	       {
+		    E T7d, T7h, T7l, T7n;
+		    T7d = W[8];
+		    T7h = W[9];
+		    rio[WS(vs, 5) + WS(is, 5)] = FMA(T7d, T7g, T7h * T7k);
+		    iio[WS(vs, 5) + WS(is, 5)] = FNMS(T7h, T7g, T7d * T7k);
+		    T7l = W[0];
+		    T7n = W[1];
+		    rio[WS(vs, 1) + WS(is, 5)] = FMA(T7l, T7m, T7n * T7o);
+		    iio[WS(vs, 1) + WS(is, 5)] = FNMS(T7n, T7m, T7l * T7o);
+	       }
+	  }
+	  {
+	       E T8g, T8s, T8q, T8u, T8f, T8p;
+	       T8f = KP707106781 * (T89 - T8e);
+	       T8g = T84 - T8f;
+	       T8s = T84 + T8f;
+	       T8p = KP707106781 * (T8n - T8o);
+	       T8q = T8m - T8p;
+	       T8u = T8m + T8p;
+	       {
+		    E T7Z, T8h, T8r, T8t;
+		    T7Z = W[12];
+		    T8h = W[13];
+		    iio[WS(vs, 7) + WS(is, 6)] = FNMS(T8h, T8q, T7Z * T8g);
+		    rio[WS(vs, 7) + WS(is, 6)] = FMA(T8h, T8g, T7Z * T8q);
+		    T8r = W[4];
+		    T8t = W[5];
+		    iio[WS(vs, 3) + WS(is, 6)] = FNMS(T8t, T8u, T8r * T8s);
+		    rio[WS(vs, 3) + WS(is, 6)] = FMA(T8t, T8s, T8r * T8u);
+	       }
+	  }
+	  {
+	       E T4G, T4M, T4K, T4O, T4F, T4J;
+	       T4F = KP707106781 * (T4m + T4h);
+	       T4G = T4E - T4F;
+	       T4M = T4E + T4F;
+	       T4J = KP707106781 * (T4v + T4w);
+	       T4K = T4I - T4J;
+	       T4O = T4I + T4J;
+	       {
+		    E T4D, T4H, T4L, T4N;
+		    T4D = W[8];
+		    T4H = W[9];
+		    rio[WS(vs, 5) + WS(is, 3)] = FMA(T4D, T4G, T4H * T4K);
+		    iio[WS(vs, 5) + WS(is, 3)] = FNMS(T4H, T4G, T4D * T4K);
+		    T4L = W[0];
+		    T4N = W[1];
+		    rio[WS(vs, 1) + WS(is, 3)] = FMA(T4L, T4M, T4N * T4O);
+		    iio[WS(vs, 1) + WS(is, 3)] = FNMS(T4N, T4M, T4L * T4O);
+	       }
+	  }
+	  {
+	       E TO, TU, TS, TW, TN, TR;
+	       TN = KP707106781 * (Tu + Tp);
+	       TO = TM - TN;
+	       TU = TM + TN;
+	       TR = KP707106781 * (TD + TE);
+	       TS = TQ - TR;
+	       TW = TQ + TR;
+	       {
+		    E TL, TP, TT, TV;
+		    TL = W[8];
+		    TP = W[9];
+		    rio[WS(vs, 5)] = FMA(TL, TO, TP * TS);
+		    iio[WS(vs, 5)] = FNMS(TP, TO, TL * TS);
+		    TT = W[0];
+		    TV = W[1];
+		    rio[WS(vs, 1)] = FMA(TT, TU, TV * TW);
+		    iio[WS(vs, 1)] = FNMS(TV, TU, TT * TW);
+	       }
+	  }
+	  {
+	       E T26, T2c, T2a, T2e, T25, T29;
+	       T25 = KP707106781 * (T1M + T1H);
+	       T26 = T24 - T25;
+	       T2c = T24 + T25;
+	       T29 = KP707106781 * (T1V + T1W);
+	       T2a = T28 - T29;
+	       T2e = T28 + T29;
+	       {
+		    E T23, T27, T2b, T2d;
+		    T23 = W[8];
+		    T27 = W[9];
+		    rio[WS(vs, 5) + WS(is, 1)] = FMA(T23, T26, T27 * T2a);
+		    iio[WS(vs, 5) + WS(is, 1)] = FNMS(T27, T26, T23 * T2a);
+		    T2b = W[0];
+		    T2d = W[1];
+		    rio[WS(vs, 1) + WS(is, 1)] = FMA(T2b, T2c, T2d * T2e);
+		    iio[WS(vs, 1) + WS(is, 1)] = FNMS(T2d, T2c, T2b * T2e);
+	       }
+	  }
+	  {
+	       E T9y, T9K, T9I, T9M, T9x, T9H;
+	       T9x = KP707106781 * (T9r - T9w);
+	       T9y = T9m - T9x;
+	       T9K = T9m + T9x;
+	       T9H = KP707106781 * (T9F - T9G);
+	       T9I = T9E - T9H;
+	       T9M = T9E + T9H;
+	       {
+		    E T9h, T9z, T9J, T9L;
+		    T9h = W[12];
+		    T9z = W[13];
+		    iio[WS(vs, 7) + WS(is, 7)] = FNMS(T9z, T9I, T9h * T9y);
+		    rio[WS(vs, 7) + WS(is, 7)] = FMA(T9z, T9y, T9h * T9I);
+		    T9J = W[4];
+		    T9L = W[5];
+		    iio[WS(vs, 3) + WS(is, 7)] = FNMS(T9L, T9M, T9J * T9K);
+		    rio[WS(vs, 3) + WS(is, 7)] = FMA(T9L, T9K, T9J * T9M);
+	       }
+	  }
+	  {
+	       E T6Y, T7a, T78, T7c, T6X, T77;
+	       T6X = KP707106781 * (T6R - T6W);
+	       T6Y = T6M - T6X;
+	       T7a = T6M + T6X;
+	       T77 = KP707106781 * (T75 - T76);
+	       T78 = T74 - T77;
+	       T7c = T74 + T77;
+	       {
+		    E T6H, T6Z, T79, T7b;
+		    T6H = W[12];
+		    T6Z = W[13];
+		    iio[WS(vs, 7) + WS(is, 5)] = FNMS(T6Z, T78, T6H * T6Y);
+		    rio[WS(vs, 7) + WS(is, 5)] = FMA(T6Z, T6Y, T6H * T78);
+		    T79 = W[4];
+		    T7b = W[5];
+		    iio[WS(vs, 3) + WS(is, 5)] = FNMS(T7b, T7c, T79 * T7a);
+		    rio[WS(vs, 3) + WS(is, 5)] = FMA(T7b, T7a, T79 * T7c);
+	       }
+	  }
+	  {
+	       E T1O, T20, T1Y, T22, T1N, T1X;
+	       T1N = KP707106781 * (T1H - T1M);
+	       T1O = T1C - T1N;
+	       T20 = T1C + T1N;
+	       T1X = KP707106781 * (T1V - T1W);
+	       T1Y = T1U - T1X;
+	       T22 = T1U + T1X;
+	       {
+		    E T1x, T1P, T1Z, T21;
+		    T1x = W[12];
+		    T1P = W[13];
+		    iio[WS(vs, 7) + WS(is, 1)] = FNMS(T1P, T1Y, T1x * T1O);
+		    rio[WS(vs, 7) + WS(is, 1)] = FMA(T1P, T1O, T1x * T1Y);
+		    T1Z = W[4];
+		    T21 = W[5];
+		    iio[WS(vs, 3) + WS(is, 1)] = FNMS(T21, T22, T1Z * T20);
+		    rio[WS(vs, 3) + WS(is, 1)] = FMA(T21, T20, T1Z * T22);
+	       }
+	  }
+	  {
+	       E T4o, T4A, T4y, T4C, T4n, T4x;
+	       T4n = KP707106781 * (T4h - T4m);
+	       T4o = T4c - T4n;
+	       T4A = T4c + T4n;
+	       T4x = KP707106781 * (T4v - T4w);
+	       T4y = T4u - T4x;
+	       T4C = T4u + T4x;
+	       {
+		    E T47, T4p, T4z, T4B;
+		    T47 = W[12];
+		    T4p = W[13];
+		    iio[WS(vs, 7) + WS(is, 3)] = FNMS(T4p, T4y, T47 * T4o);
+		    rio[WS(vs, 7) + WS(is, 3)] = FMA(T4p, T4o, T47 * T4y);
+		    T4z = W[4];
+		    T4B = W[5];
+		    iio[WS(vs, 3) + WS(is, 3)] = FNMS(T4B, T4C, T4z * T4A);
+		    rio[WS(vs, 3) + WS(is, 3)] = FMA(T4B, T4A, T4z * T4C);
+	       }
+	  }
+	  {
+	       E T3o, T3u, T3s, T3w, T3n, T3r;
+	       T3n = KP707106781 * (T34 + T2Z);
+	       T3o = T3m - T3n;
+	       T3u = T3m + T3n;
+	       T3r = KP707106781 * (T3d + T3e);
+	       T3s = T3q - T3r;
+	       T3w = T3q + T3r;
+	       {
+		    E T3l, T3p, T3t, T3v;
+		    T3l = W[8];
+		    T3p = W[9];
+		    rio[WS(vs, 5) + WS(is, 2)] = FMA(T3l, T3o, T3p * T3s);
+		    iio[WS(vs, 5) + WS(is, 2)] = FNMS(T3p, T3o, T3l * T3s);
+		    T3t = W[0];
+		    T3v = W[1];
+		    rio[WS(vs, 1) + WS(is, 2)] = FMA(T3t, T3u, T3v * T3w);
+		    iio[WS(vs, 1) + WS(is, 2)] = FNMS(T3v, T3u, T3t * T3w);
+	       }
+	  }
+	  {
+	       E T8y, T8E, T8C, T8G, T8x, T8B;
+	       T8x = KP707106781 * (T8e + T89);
+	       T8y = T8w - T8x;
+	       T8E = T8w + T8x;
+	       T8B = KP707106781 * (T8n + T8o);
+	       T8C = T8A - T8B;
+	       T8G = T8A + T8B;
+	       {
+		    E T8v, T8z, T8D, T8F;
+		    T8v = W[8];
+		    T8z = W[9];
+		    rio[WS(vs, 5) + WS(is, 6)] = FMA(T8v, T8y, T8z * T8C);
+		    iio[WS(vs, 5) + WS(is, 6)] = FNMS(T8z, T8y, T8v * T8C);
+		    T8D = W[0];
+		    T8F = W[1];
+		    rio[WS(vs, 1) + WS(is, 6)] = FMA(T8D, T8E, T8F * T8G);
+		    iio[WS(vs, 1) + WS(is, 6)] = FNMS(T8F, T8E, T8D * T8G);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 8},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 8, "q1_8", twinstr, {416, 144, 112, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_q1_8) (planner *p) {
+     X(kdft_difsq_register) (p, q1_8, &desc);
+}
diff --git a/src/fftw3/dft/codelets/n.c b/src/fftw3/dft/codelets/n.c
new file mode 100644
index 0000000..22f0528
--- /dev/null
+++ b/src/fftw3/dft/codelets/n.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "codelet-dft.h"
+#include "n.h"
+
+static int okp(const kdft_desc *d,
+	       const R *ri, const R *ii, 
+	       const R *ro, const R *io,
+	       int is, int os, int vl, int ivs, int ovs,
+	       const planner *plnr)
+{
+     UNUSED(ri); UNUSED(ii); UNUSED(ro); UNUSED(io); UNUSED(vl); UNUSED(plnr);
+     return (1
+	     && (!d->is || (d->is == is))
+	     && (!d->os || (d->os == os))
+	     && (!d->ivs || (d->ivs == ivs))
+	     && (!d->ovs || (d->ovs == ovs))
+	  );
+}
+
+const kdft_genus GENUS = { okp, 1 };
diff --git a/src/fftw3/dft/codelets/n.h b/src/fftw3/dft/codelets/n.h
new file mode 100644
index 0000000..249c1c9
--- /dev/null
+++ b/src/fftw3/dft/codelets/n.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+
+#define GENUS X(dft_n_genus)
+extern const kdft_genus GENUS;
diff --git a/src/fftw3/dft/codelets/q.h b/src/fftw3/dft/codelets/q.h
new file mode 100644
index 0000000..6a9dd0b
--- /dev/null
+++ b/src/fftw3/dft/codelets/q.h
@@ -0,0 +1 @@
+#include "t.h"  /* same stuff, no need to duplicate */
diff --git a/src/fftw3/dft/codelets/standard/m1_16.c b/src/fftw3/dft/codelets/standard/m1_16.c
new file mode 100644
index 0000000..9882bce
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/m1_16.c
@@ -0,0 +1,305 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:37 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw_noinline -compact -variables 4 -n 16 -name m1_16 -include n.h */
+
+/*
+ * This function contains 144 FP additions, 24 FP multiplications,
+ * (or, 136 additions, 16 multiplications, 8 fused multiply/add),
+ * 49 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: m1_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: m1_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: m1_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void m1_16_0(const R *ri, const R *ii, R *ro, R *io, stride is, stride os)
+{
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     {
+	  E T7, T1R, T25, TC, TN, T1x, T1H, T1l, Tt, T22, T2h, T1b, T1g, T1E, T1Z;
+	  E T1D, Te, T1S, T26, TJ, TQ, T1m, T1n, TT, Tm, T1X, T2g, T10, T15, T1B;
+	  E T1U, T1A;
+	  {
+	       E T3, TL, Ty, T1k, T6, T1j, TB, TM;
+	       {
+		    E T1, T2, Tw, Tx;
+		    T1 = ri[0];
+		    T2 = ri[WS(is, 8)];
+		    T3 = T1 + T2;
+		    TL = T1 - T2;
+		    Tw = ii[0];
+		    Tx = ii[WS(is, 8)];
+		    Ty = Tw + Tx;
+		    T1k = Tw - Tx;
+	       }
+	       {
+		    E T4, T5, Tz, TA;
+		    T4 = ri[WS(is, 4)];
+		    T5 = ri[WS(is, 12)];
+		    T6 = T4 + T5;
+		    T1j = T4 - T5;
+		    Tz = ii[WS(is, 4)];
+		    TA = ii[WS(is, 12)];
+		    TB = Tz + TA;
+		    TM = Tz - TA;
+	       }
+	       T7 = T3 + T6;
+	       T1R = T3 - T6;
+	       T25 = Ty - TB;
+	       TC = Ty + TB;
+	       TN = TL - TM;
+	       T1x = TL + TM;
+	       T1H = T1k - T1j;
+	       T1l = T1j + T1k;
+	  }
+	  {
+	       E Tp, T17, T1f, T20, Ts, T1c, T1a, T21;
+	       {
+		    E Tn, To, T1d, T1e;
+		    Tn = ri[WS(is, 15)];
+		    To = ri[WS(is, 7)];
+		    Tp = Tn + To;
+		    T17 = Tn - To;
+		    T1d = ii[WS(is, 15)];
+		    T1e = ii[WS(is, 7)];
+		    T1f = T1d - T1e;
+		    T20 = T1d + T1e;
+	       }
+	       {
+		    E Tq, Tr, T18, T19;
+		    Tq = ri[WS(is, 3)];
+		    Tr = ri[WS(is, 11)];
+		    Ts = Tq + Tr;
+		    T1c = Tq - Tr;
+		    T18 = ii[WS(is, 3)];
+		    T19 = ii[WS(is, 11)];
+		    T1a = T18 - T19;
+		    T21 = T18 + T19;
+	       }
+	       Tt = Tp + Ts;
+	       T22 = T20 - T21;
+	       T2h = T20 + T21;
+	       T1b = T17 - T1a;
+	       T1g = T1c + T1f;
+	       T1E = T1f - T1c;
+	       T1Z = Tp - Ts;
+	       T1D = T17 + T1a;
+	  }
+	  {
+	       E Ta, TP, TF, TO, Td, TR, TI, TS;
+	       {
+		    E T8, T9, TD, TE;
+		    T8 = ri[WS(is, 2)];
+		    T9 = ri[WS(is, 10)];
+		    Ta = T8 + T9;
+		    TP = T8 - T9;
+		    TD = ii[WS(is, 2)];
+		    TE = ii[WS(is, 10)];
+		    TF = TD + TE;
+		    TO = TD - TE;
+	       }
+	       {
+		    E Tb, Tc, TG, TH;
+		    Tb = ri[WS(is, 14)];
+		    Tc = ri[WS(is, 6)];
+		    Td = Tb + Tc;
+		    TR = Tb - Tc;
+		    TG = ii[WS(is, 14)];
+		    TH = ii[WS(is, 6)];
+		    TI = TG + TH;
+		    TS = TG - TH;
+	       }
+	       Te = Ta + Td;
+	       T1S = TF - TI;
+	       T26 = Td - Ta;
+	       TJ = TF + TI;
+	       TQ = TO - TP;
+	       T1m = TR - TS;
+	       T1n = TP + TO;
+	       TT = TR + TS;
+	  }
+	  {
+	       E Ti, T11, TZ, T1V, Tl, TW, T14, T1W;
+	       {
+		    E Tg, Th, TX, TY;
+		    Tg = ri[WS(is, 1)];
+		    Th = ri[WS(is, 9)];
+		    Ti = Tg + Th;
+		    T11 = Tg - Th;
+		    TX = ii[WS(is, 1)];
+		    TY = ii[WS(is, 9)];
+		    TZ = TX - TY;
+		    T1V = TX + TY;
+	       }
+	       {
+		    E Tj, Tk, T12, T13;
+		    Tj = ri[WS(is, 5)];
+		    Tk = ri[WS(is, 13)];
+		    Tl = Tj + Tk;
+		    TW = Tj - Tk;
+		    T12 = ii[WS(is, 5)];
+		    T13 = ii[WS(is, 13)];
+		    T14 = T12 - T13;
+		    T1W = T12 + T13;
+	       }
+	       Tm = Ti + Tl;
+	       T1X = T1V - T1W;
+	       T2g = T1V + T1W;
+	       T10 = TW + TZ;
+	       T15 = T11 - T14;
+	       T1B = T11 + T14;
+	       T1U = Ti - Tl;
+	       T1A = TZ - TW;
+	  }
+	  {
+	       E Tf, Tu, T2j, T2k;
+	       Tf = T7 + Te;
+	       Tu = Tm + Tt;
+	       ro[WS(os, 8)] = Tf - Tu;
+	       ro[0] = Tf + Tu;
+	       T2j = TC + TJ;
+	       T2k = T2g + T2h;
+	       io[WS(os, 8)] = T2j - T2k;
+	       io[0] = T2j + T2k;
+	  }
+	  {
+	       E Tv, TK, T2f, T2i;
+	       Tv = Tt - Tm;
+	       TK = TC - TJ;
+	       io[WS(os, 4)] = Tv + TK;
+	       io[WS(os, 12)] = TK - Tv;
+	       T2f = T7 - Te;
+	       T2i = T2g - T2h;
+	       ro[WS(os, 12)] = T2f - T2i;
+	       ro[WS(os, 4)] = T2f + T2i;
+	  }
+	  {
+	       E T1T, T27, T24, T28, T1Y, T23;
+	       T1T = T1R + T1S;
+	       T27 = T25 - T26;
+	       T1Y = T1U + T1X;
+	       T23 = T1Z - T22;
+	       T24 = KP707106781 * (T1Y + T23);
+	       T28 = KP707106781 * (T23 - T1Y);
+	       ro[WS(os, 10)] = T1T - T24;
+	       io[WS(os, 6)] = T27 + T28;
+	       ro[WS(os, 2)] = T1T + T24;
+	       io[WS(os, 14)] = T27 - T28;
+	  }
+	  {
+	       E T29, T2d, T2c, T2e, T2a, T2b;
+	       T29 = T1R - T1S;
+	       T2d = T26 + T25;
+	       T2a = T1X - T1U;
+	       T2b = T1Z + T22;
+	       T2c = KP707106781 * (T2a - T2b);
+	       T2e = KP707106781 * (T2a + T2b);
+	       ro[WS(os, 14)] = T29 - T2c;
+	       io[WS(os, 2)] = T2d + T2e;
+	       ro[WS(os, 6)] = T29 + T2c;
+	       io[WS(os, 10)] = T2d - T2e;
+	  }
+	  {
+	       E TV, T1r, T1p, T1v, T1i, T1q, T1u, T1w, TU, T1o;
+	       TU = KP707106781 * (TQ - TT);
+	       TV = TN + TU;
+	       T1r = TN - TU;
+	       T1o = KP707106781 * (T1m - T1n);
+	       T1p = T1l - T1o;
+	       T1v = T1l + T1o;
+	       {
+		    E T16, T1h, T1s, T1t;
+		    T16 = FMA(KP923879532, T10, KP382683432 * T15);
+		    T1h = FNMS(KP923879532, T1g, KP382683432 * T1b);
+		    T1i = T16 + T1h;
+		    T1q = T1h - T16;
+		    T1s = FNMS(KP923879532, T15, KP382683432 * T10);
+		    T1t = FMA(KP382683432, T1g, KP923879532 * T1b);
+		    T1u = T1s - T1t;
+		    T1w = T1s + T1t;
+	       }
+	       ro[WS(os, 11)] = TV - T1i;
+	       io[WS(os, 11)] = T1v - T1w;
+	       ro[WS(os, 3)] = TV + T1i;
+	       io[WS(os, 3)] = T1v + T1w;
+	       io[WS(os, 15)] = T1p - T1q;
+	       ro[WS(os, 15)] = T1r - T1u;
+	       io[WS(os, 7)] = T1p + T1q;
+	       ro[WS(os, 7)] = T1r + T1u;
+	  }
+	  {
+	       E T1z, T1L, T1J, T1P, T1G, T1K, T1O, T1Q, T1y, T1I;
+	       T1y = KP707106781 * (T1n + T1m);
+	       T1z = T1x + T1y;
+	       T1L = T1x - T1y;
+	       T1I = KP707106781 * (TQ + TT);
+	       T1J = T1H - T1I;
+	       T1P = T1H + T1I;
+	       {
+		    E T1C, T1F, T1M, T1N;
+		    T1C = FMA(KP382683432, T1A, KP923879532 * T1B);
+		    T1F = FNMS(KP382683432, T1E, KP923879532 * T1D);
+		    T1G = T1C + T1F;
+		    T1K = T1F - T1C;
+		    T1M = FNMS(KP382683432, T1B, KP923879532 * T1A);
+		    T1N = FMA(KP923879532, T1E, KP382683432 * T1D);
+		    T1O = T1M - T1N;
+		    T1Q = T1M + T1N;
+	       }
+	       ro[WS(os, 9)] = T1z - T1G;
+	       io[WS(os, 9)] = T1P - T1Q;
+	       ro[WS(os, 1)] = T1z + T1G;
+	       io[WS(os, 1)] = T1P + T1Q;
+	       io[WS(os, 13)] = T1J - T1K;
+	       ro[WS(os, 13)] = T1L - T1O;
+	       io[WS(os, 5)] = T1J + T1K;
+	       ro[WS(os, 5)] = T1L + T1O;
+	  }
+     }
+}
+
+static void m1_16(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; i -= 1) {
+	  m1_16_0(ri, ii, ro, io, is, os);
+	  ri += ivs;
+	  ii += ivs;
+	  ro += ovs;
+	  io += ovs;
+     }
+}
+
+static const kdft_desc desc = { 16, "m1_16", {136, 16, 8, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_m1_16) (planner *p) {
+     X(kdft_register) (p, m1_16, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/m1_32.c b/src/fftw3/dft/codelets/standard/m1_32.c
new file mode 100644
index 0000000..aa75d21
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/m1_32.c
@@ -0,0 +1,684 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:38 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw_noinline -compact -variables 4 -n 32 -name m1_32 -include n.h */
+
+/*
+ * This function contains 372 FP additions, 84 FP multiplications,
+ * (or, 340 additions, 52 multiplications, 32 fused multiply/add),
+ * 99 stack variables, and 128 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: m1_32.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: m1_32.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: m1_32.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void m1_32_0(const R *ri, const R *ii, R *ro, R *io, stride is, stride os)
+{
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     {
+	  E T7, T4r, T4Z, T18, T1z, T3t, T3T, T2T, Te, T1f, T50, T4s, T2W, T3u, T1G;
+	  E T3U, Tm, T1n, T1O, T2Z, T3y, T3X, T4w, T53, Tt, T1u, T1V, T2Y, T3B, T3W;
+	  E T4z, T52, T2t, T3L, T3O, T2K, TR, TY, T5F, T5G, T5H, T5I, T4R, T5j, T2E;
+	  E T3P, T4W, T5k, T2N, T3M, T22, T3E, T3H, T2j, TC, TJ, T5A, T5B, T5C, T5D;
+	  E T4G, T5g, T2d, T3F, T4L, T5h, T2m, T3I;
+	  {
+	       E T3, T1x, T14, T2S, T6, T2R, T17, T1y;
+	       {
+		    E T1, T2, T12, T13;
+		    T1 = ri[0];
+		    T2 = ri[WS(is, 16)];
+		    T3 = T1 + T2;
+		    T1x = T1 - T2;
+		    T12 = ii[0];
+		    T13 = ii[WS(is, 16)];
+		    T14 = T12 + T13;
+		    T2S = T12 - T13;
+	       }
+	       {
+		    E T4, T5, T15, T16;
+		    T4 = ri[WS(is, 8)];
+		    T5 = ri[WS(is, 24)];
+		    T6 = T4 + T5;
+		    T2R = T4 - T5;
+		    T15 = ii[WS(is, 8)];
+		    T16 = ii[WS(is, 24)];
+		    T17 = T15 + T16;
+		    T1y = T15 - T16;
+	       }
+	       T7 = T3 + T6;
+	       T4r = T3 - T6;
+	       T4Z = T14 - T17;
+	       T18 = T14 + T17;
+	       T1z = T1x - T1y;
+	       T3t = T1x + T1y;
+	       T3T = T2S - T2R;
+	       T2T = T2R + T2S;
+	  }
+	  {
+	       E Ta, T1B, T1b, T1A, Td, T1D, T1e, T1E;
+	       {
+		    E T8, T9, T19, T1a;
+		    T8 = ri[WS(is, 4)];
+		    T9 = ri[WS(is, 20)];
+		    Ta = T8 + T9;
+		    T1B = T8 - T9;
+		    T19 = ii[WS(is, 4)];
+		    T1a = ii[WS(is, 20)];
+		    T1b = T19 + T1a;
+		    T1A = T19 - T1a;
+	       }
+	       {
+		    E Tb, Tc, T1c, T1d;
+		    Tb = ri[WS(is, 28)];
+		    Tc = ri[WS(is, 12)];
+		    Td = Tb + Tc;
+		    T1D = Tb - Tc;
+		    T1c = ii[WS(is, 28)];
+		    T1d = ii[WS(is, 12)];
+		    T1e = T1c + T1d;
+		    T1E = T1c - T1d;
+	       }
+	       Te = Ta + Td;
+	       T1f = T1b + T1e;
+	       T50 = Td - Ta;
+	       T4s = T1b - T1e;
+	       {
+		    E T2U, T2V, T1C, T1F;
+		    T2U = T1D - T1E;
+		    T2V = T1B + T1A;
+		    T2W = KP707106781 * (T2U - T2V);
+		    T3u = KP707106781 * (T2V + T2U);
+		    T1C = T1A - T1B;
+		    T1F = T1D + T1E;
+		    T1G = KP707106781 * (T1C - T1F);
+		    T3U = KP707106781 * (T1C + T1F);
+	       }
+	  }
+	  {
+	       E Ti, T1L, T1j, T1J, Tl, T1I, T1m, T1M, T1K, T1N;
+	       {
+		    E Tg, Th, T1h, T1i;
+		    Tg = ri[WS(is, 2)];
+		    Th = ri[WS(is, 18)];
+		    Ti = Tg + Th;
+		    T1L = Tg - Th;
+		    T1h = ii[WS(is, 2)];
+		    T1i = ii[WS(is, 18)];
+		    T1j = T1h + T1i;
+		    T1J = T1h - T1i;
+	       }
+	       {
+		    E Tj, Tk, T1k, T1l;
+		    Tj = ri[WS(is, 10)];
+		    Tk = ri[WS(is, 26)];
+		    Tl = Tj + Tk;
+		    T1I = Tj - Tk;
+		    T1k = ii[WS(is, 10)];
+		    T1l = ii[WS(is, 26)];
+		    T1m = T1k + T1l;
+		    T1M = T1k - T1l;
+	       }
+	       Tm = Ti + Tl;
+	       T1n = T1j + T1m;
+	       T1K = T1I + T1J;
+	       T1N = T1L - T1M;
+	       T1O = FNMS(KP923879532, T1N, KP382683432 * T1K);
+	       T2Z = FMA(KP923879532, T1K, KP382683432 * T1N);
+	       {
+		    E T3w, T3x, T4u, T4v;
+		    T3w = T1J - T1I;
+		    T3x = T1L + T1M;
+		    T3y = FNMS(KP382683432, T3x, KP923879532 * T3w);
+		    T3X = FMA(KP382683432, T3w, KP923879532 * T3x);
+		    T4u = T1j - T1m;
+		    T4v = Ti - Tl;
+		    T4w = T4u - T4v;
+		    T53 = T4v + T4u;
+	       }
+	  }
+	  {
+	       E Tp, T1S, T1q, T1Q, Ts, T1P, T1t, T1T, T1R, T1U;
+	       {
+		    E Tn, To, T1o, T1p;
+		    Tn = ri[WS(is, 30)];
+		    To = ri[WS(is, 14)];
+		    Tp = Tn + To;
+		    T1S = Tn - To;
+		    T1o = ii[WS(is, 30)];
+		    T1p = ii[WS(is, 14)];
+		    T1q = T1o + T1p;
+		    T1Q = T1o - T1p;
+	       }
+	       {
+		    E Tq, Tr, T1r, T1s;
+		    Tq = ri[WS(is, 6)];
+		    Tr = ri[WS(is, 22)];
+		    Ts = Tq + Tr;
+		    T1P = Tq - Tr;
+		    T1r = ii[WS(is, 6)];
+		    T1s = ii[WS(is, 22)];
+		    T1t = T1r + T1s;
+		    T1T = T1r - T1s;
+	       }
+	       Tt = Tp + Ts;
+	       T1u = T1q + T1t;
+	       T1R = T1P + T1Q;
+	       T1U = T1S - T1T;
+	       T1V = FMA(KP382683432, T1R, KP923879532 * T1U);
+	       T2Y = FNMS(KP923879532, T1R, KP382683432 * T1U);
+	       {
+		    E T3z, T3A, T4x, T4y;
+		    T3z = T1Q - T1P;
+		    T3A = T1S + T1T;
+		    T3B = FMA(KP923879532, T3z, KP382683432 * T3A);
+		    T3W = FNMS(KP382683432, T3z, KP923879532 * T3A);
+		    T4x = Tp - Ts;
+		    T4y = T1q - T1t;
+		    T4z = T4x + T4y;
+		    T52 = T4x - T4y;
+	       }
+	  }
+	  {
+	       E TN, T2p, T2J, T4S, TQ, T2G, T2s, T4T, TU, T2x, T2w, T4O, TX, T2z, T2C;
+	       E T4P;
+	       {
+		    E TL, TM, T2H, T2I;
+		    TL = ri[WS(is, 31)];
+		    TM = ri[WS(is, 15)];
+		    TN = TL + TM;
+		    T2p = TL - TM;
+		    T2H = ii[WS(is, 31)];
+		    T2I = ii[WS(is, 15)];
+		    T2J = T2H - T2I;
+		    T4S = T2H + T2I;
+	       }
+	       {
+		    E TO, TP, T2q, T2r;
+		    TO = ri[WS(is, 7)];
+		    TP = ri[WS(is, 23)];
+		    TQ = TO + TP;
+		    T2G = TO - TP;
+		    T2q = ii[WS(is, 7)];
+		    T2r = ii[WS(is, 23)];
+		    T2s = T2q - T2r;
+		    T4T = T2q + T2r;
+	       }
+	       {
+		    E TS, TT, T2u, T2v;
+		    TS = ri[WS(is, 3)];
+		    TT = ri[WS(is, 19)];
+		    TU = TS + TT;
+		    T2x = TS - TT;
+		    T2u = ii[WS(is, 3)];
+		    T2v = ii[WS(is, 19)];
+		    T2w = T2u - T2v;
+		    T4O = T2u + T2v;
+	       }
+	       {
+		    E TV, TW, T2A, T2B;
+		    TV = ri[WS(is, 27)];
+		    TW = ri[WS(is, 11)];
+		    TX = TV + TW;
+		    T2z = TV - TW;
+		    T2A = ii[WS(is, 27)];
+		    T2B = ii[WS(is, 11)];
+		    T2C = T2A - T2B;
+		    T4P = T2A + T2B;
+	       }
+	       T2t = T2p - T2s;
+	       T3L = T2p + T2s;
+	       T3O = T2J - T2G;
+	       T2K = T2G + T2J;
+	       TR = TN + TQ;
+	       TY = TU + TX;
+	       T5F = TR - TY;
+	       {
+		    E T4N, T4Q, T2y, T2D;
+		    T5G = T4S + T4T;
+		    T5H = T4O + T4P;
+		    T5I = T5G - T5H;
+		    T4N = TN - TQ;
+		    T4Q = T4O - T4P;
+		    T4R = T4N - T4Q;
+		    T5j = T4N + T4Q;
+		    T2y = T2w - T2x;
+		    T2D = T2z + T2C;
+		    T2E = KP707106781 * (T2y - T2D);
+		    T3P = KP707106781 * (T2y + T2D);
+		    {
+			 E T4U, T4V, T2L, T2M;
+			 T4U = T4S - T4T;
+			 T4V = TX - TU;
+			 T4W = T4U - T4V;
+			 T5k = T4V + T4U;
+			 T2L = T2z - T2C;
+			 T2M = T2x + T2w;
+			 T2N = KP707106781 * (T2L - T2M);
+			 T3M = KP707106781 * (T2M + T2L);
+		    }
+	       }
+	  }
+	  {
+	       E Ty, T2f, T21, T4C, TB, T1Y, T2i, T4D, TF, T28, T2b, T4I, TI, T23, T26;
+	       E T4J;
+	       {
+		    E Tw, Tx, T1Z, T20;
+		    Tw = ri[WS(is, 1)];
+		    Tx = ri[WS(is, 17)];
+		    Ty = Tw + Tx;
+		    T2f = Tw - Tx;
+		    T1Z = ii[WS(is, 1)];
+		    T20 = ii[WS(is, 17)];
+		    T21 = T1Z - T20;
+		    T4C = T1Z + T20;
+	       }
+	       {
+		    E Tz, TA, T2g, T2h;
+		    Tz = ri[WS(is, 9)];
+		    TA = ri[WS(is, 25)];
+		    TB = Tz + TA;
+		    T1Y = Tz - TA;
+		    T2g = ii[WS(is, 9)];
+		    T2h = ii[WS(is, 25)];
+		    T2i = T2g - T2h;
+		    T4D = T2g + T2h;
+	       }
+	       {
+		    E TD, TE, T29, T2a;
+		    TD = ri[WS(is, 5)];
+		    TE = ri[WS(is, 21)];
+		    TF = TD + TE;
+		    T28 = TD - TE;
+		    T29 = ii[WS(is, 5)];
+		    T2a = ii[WS(is, 21)];
+		    T2b = T29 - T2a;
+		    T4I = T29 + T2a;
+	       }
+	       {
+		    E TG, TH, T24, T25;
+		    TG = ri[WS(is, 29)];
+		    TH = ri[WS(is, 13)];
+		    TI = TG + TH;
+		    T23 = TG - TH;
+		    T24 = ii[WS(is, 29)];
+		    T25 = ii[WS(is, 13)];
+		    T26 = T24 - T25;
+		    T4J = T24 + T25;
+	       }
+	       T22 = T1Y + T21;
+	       T3E = T2f + T2i;
+	       T3H = T21 - T1Y;
+	       T2j = T2f - T2i;
+	       TC = Ty + TB;
+	       TJ = TF + TI;
+	       T5A = TC - TJ;
+	       {
+		    E T4E, T4F, T27, T2c;
+		    T5B = T4C + T4D;
+		    T5C = T4I + T4J;
+		    T5D = T5B - T5C;
+		    T4E = T4C - T4D;
+		    T4F = TI - TF;
+		    T4G = T4E - T4F;
+		    T5g = T4F + T4E;
+		    T27 = T23 - T26;
+		    T2c = T28 + T2b;
+		    T2d = KP707106781 * (T27 - T2c);
+		    T3F = KP707106781 * (T2c + T27);
+		    {
+			 E T4H, T4K, T2k, T2l;
+			 T4H = Ty - TB;
+			 T4K = T4I - T4J;
+			 T4L = T4H - T4K;
+			 T5h = T4H + T4K;
+			 T2k = T2b - T28;
+			 T2l = T23 + T26;
+			 T2m = KP707106781 * (T2k - T2l);
+			 T3I = KP707106781 * (T2k + T2l);
+		    }
+	       }
+	  }
+	  {
+	       E T4B, T57, T5a, T5c, T4Y, T56, T55, T5b;
+	       {
+		    E T4t, T4A, T58, T59;
+		    T4t = T4r - T4s;
+		    T4A = KP707106781 * (T4w - T4z);
+		    T4B = T4t + T4A;
+		    T57 = T4t - T4A;
+		    T58 = FNMS(KP923879532, T4L, KP382683432 * T4G);
+		    T59 = FMA(KP382683432, T4W, KP923879532 * T4R);
+		    T5a = T58 - T59;
+		    T5c = T58 + T59;
+	       }
+	       {
+		    E T4M, T4X, T51, T54;
+		    T4M = FMA(KP923879532, T4G, KP382683432 * T4L);
+		    T4X = FNMS(KP923879532, T4W, KP382683432 * T4R);
+		    T4Y = T4M + T4X;
+		    T56 = T4X - T4M;
+		    T51 = T4Z - T50;
+		    T54 = KP707106781 * (T52 - T53);
+		    T55 = T51 - T54;
+		    T5b = T51 + T54;
+	       }
+	       ro[WS(os, 22)] = T4B - T4Y;
+	       io[WS(os, 22)] = T5b - T5c;
+	       ro[WS(os, 6)] = T4B + T4Y;
+	       io[WS(os, 6)] = T5b + T5c;
+	       io[WS(os, 30)] = T55 - T56;
+	       ro[WS(os, 30)] = T57 - T5a;
+	       io[WS(os, 14)] = T55 + T56;
+	       ro[WS(os, 14)] = T57 + T5a;
+	  }
+	  {
+	       E T5f, T5r, T5u, T5w, T5m, T5q, T5p, T5v;
+	       {
+		    E T5d, T5e, T5s, T5t;
+		    T5d = T4r + T4s;
+		    T5e = KP707106781 * (T53 + T52);
+		    T5f = T5d + T5e;
+		    T5r = T5d - T5e;
+		    T5s = FNMS(KP382683432, T5h, KP923879532 * T5g);
+		    T5t = FMA(KP923879532, T5k, KP382683432 * T5j);
+		    T5u = T5s - T5t;
+		    T5w = T5s + T5t;
+	       }
+	       {
+		    E T5i, T5l, T5n, T5o;
+		    T5i = FMA(KP382683432, T5g, KP923879532 * T5h);
+		    T5l = FNMS(KP382683432, T5k, KP923879532 * T5j);
+		    T5m = T5i + T5l;
+		    T5q = T5l - T5i;
+		    T5n = T50 + T4Z;
+		    T5o = KP707106781 * (T4w + T4z);
+		    T5p = T5n - T5o;
+		    T5v = T5n + T5o;
+	       }
+	       ro[WS(os, 18)] = T5f - T5m;
+	       io[WS(os, 18)] = T5v - T5w;
+	       ro[WS(os, 2)] = T5f + T5m;
+	       io[WS(os, 2)] = T5v + T5w;
+	       io[WS(os, 26)] = T5p - T5q;
+	       ro[WS(os, 26)] = T5r - T5u;
+	       io[WS(os, 10)] = T5p + T5q;
+	       ro[WS(os, 10)] = T5r + T5u;
+	  }
+	  {
+	       E T5z, T5P, T5S, T5U, T5K, T5O, T5N, T5T;
+	       {
+		    E T5x, T5y, T5Q, T5R;
+		    T5x = T7 - Te;
+		    T5y = T1n - T1u;
+		    T5z = T5x + T5y;
+		    T5P = T5x - T5y;
+		    T5Q = T5D - T5A;
+		    T5R = T5F + T5I;
+		    T5S = KP707106781 * (T5Q - T5R);
+		    T5U = KP707106781 * (T5Q + T5R);
+	       }
+	       {
+		    E T5E, T5J, T5L, T5M;
+		    T5E = T5A + T5D;
+		    T5J = T5F - T5I;
+		    T5K = KP707106781 * (T5E + T5J);
+		    T5O = KP707106781 * (T5J - T5E);
+		    T5L = T18 - T1f;
+		    T5M = Tt - Tm;
+		    T5N = T5L - T5M;
+		    T5T = T5M + T5L;
+	       }
+	       ro[WS(os, 20)] = T5z - T5K;
+	       io[WS(os, 20)] = T5T - T5U;
+	       ro[WS(os, 4)] = T5z + T5K;
+	       io[WS(os, 4)] = T5T + T5U;
+	       io[WS(os, 28)] = T5N - T5O;
+	       ro[WS(os, 28)] = T5P - T5S;
+	       io[WS(os, 12)] = T5N + T5O;
+	       ro[WS(os, 12)] = T5P + T5S;
+	  }
+	  {
+	       E Tv, T5V, T5Y, T60, T10, T11, T1w, T5Z;
+	       {
+		    E Tf, Tu, T5W, T5X;
+		    Tf = T7 + Te;
+		    Tu = Tm + Tt;
+		    Tv = Tf + Tu;
+		    T5V = Tf - Tu;
+		    T5W = T5B + T5C;
+		    T5X = T5G + T5H;
+		    T5Y = T5W - T5X;
+		    T60 = T5W + T5X;
+	       }
+	       {
+		    E TK, TZ, T1g, T1v;
+		    TK = TC + TJ;
+		    TZ = TR + TY;
+		    T10 = TK + TZ;
+		    T11 = TZ - TK;
+		    T1g = T18 + T1f;
+		    T1v = T1n + T1u;
+		    T1w = T1g - T1v;
+		    T5Z = T1g + T1v;
+	       }
+	       ro[WS(os, 16)] = Tv - T10;
+	       io[WS(os, 16)] = T5Z - T60;
+	       ro[0] = Tv + T10;
+	       io[0] = T5Z + T60;
+	       io[WS(os, 8)] = T11 + T1w;
+	       ro[WS(os, 8)] = T5V + T5Y;
+	       io[WS(os, 24)] = T1w - T11;
+	       ro[WS(os, 24)] = T5V - T5Y;
+	  }
+	  {
+	       E T1X, T33, T31, T37, T2o, T34, T2P, T35;
+	       {
+		    E T1H, T1W, T2X, T30;
+		    T1H = T1z - T1G;
+		    T1W = T1O - T1V;
+		    T1X = T1H + T1W;
+		    T33 = T1H - T1W;
+		    T2X = T2T - T2W;
+		    T30 = T2Y - T2Z;
+		    T31 = T2X - T30;
+		    T37 = T2X + T30;
+	       }
+	       {
+		    E T2e, T2n, T2F, T2O;
+		    T2e = T22 - T2d;
+		    T2n = T2j - T2m;
+		    T2o = FMA(KP980785280, T2e, KP195090322 * T2n);
+		    T34 = FNMS(KP980785280, T2n, KP195090322 * T2e);
+		    T2F = T2t - T2E;
+		    T2O = T2K - T2N;
+		    T2P = FNMS(KP980785280, T2O, KP195090322 * T2F);
+		    T35 = FMA(KP195090322, T2O, KP980785280 * T2F);
+	       }
+	       {
+		    E T2Q, T38, T32, T36;
+		    T2Q = T2o + T2P;
+		    ro[WS(os, 23)] = T1X - T2Q;
+		    ro[WS(os, 7)] = T1X + T2Q;
+		    T38 = T34 + T35;
+		    io[WS(os, 23)] = T37 - T38;
+		    io[WS(os, 7)] = T37 + T38;
+		    T32 = T2P - T2o;
+		    io[WS(os, 31)] = T31 - T32;
+		    io[WS(os, 15)] = T31 + T32;
+		    T36 = T34 - T35;
+		    ro[WS(os, 31)] = T33 - T36;
+		    ro[WS(os, 15)] = T33 + T36;
+	       }
+	  }
+	  {
+	       E T3D, T41, T3Z, T45, T3K, T42, T3R, T43;
+	       {
+		    E T3v, T3C, T3V, T3Y;
+		    T3v = T3t - T3u;
+		    T3C = T3y - T3B;
+		    T3D = T3v + T3C;
+		    T41 = T3v - T3C;
+		    T3V = T3T - T3U;
+		    T3Y = T3W - T3X;
+		    T3Z = T3V - T3Y;
+		    T45 = T3V + T3Y;
+	       }
+	       {
+		    E T3G, T3J, T3N, T3Q;
+		    T3G = T3E - T3F;
+		    T3J = T3H - T3I;
+		    T3K = FMA(KP555570233, T3G, KP831469612 * T3J);
+		    T42 = FNMS(KP831469612, T3G, KP555570233 * T3J);
+		    T3N = T3L - T3M;
+		    T3Q = T3O - T3P;
+		    T3R = FNMS(KP831469612, T3Q, KP555570233 * T3N);
+		    T43 = FMA(KP831469612, T3N, KP555570233 * T3Q);
+	       }
+	       {
+		    E T3S, T46, T40, T44;
+		    T3S = T3K + T3R;
+		    ro[WS(os, 21)] = T3D - T3S;
+		    ro[WS(os, 5)] = T3D + T3S;
+		    T46 = T42 + T43;
+		    io[WS(os, 21)] = T45 - T46;
+		    io[WS(os, 5)] = T45 + T46;
+		    T40 = T3R - T3K;
+		    io[WS(os, 29)] = T3Z - T40;
+		    io[WS(os, 13)] = T3Z + T40;
+		    T44 = T42 - T43;
+		    ro[WS(os, 29)] = T41 - T44;
+		    ro[WS(os, 13)] = T41 + T44;
+	       }
+	  }
+	  {
+	       E T49, T4l, T4j, T4p, T4c, T4m, T4f, T4n;
+	       {
+		    E T47, T48, T4h, T4i;
+		    T47 = T3t + T3u;
+		    T48 = T3X + T3W;
+		    T49 = T47 + T48;
+		    T4l = T47 - T48;
+		    T4h = T3T + T3U;
+		    T4i = T3y + T3B;
+		    T4j = T4h - T4i;
+		    T4p = T4h + T4i;
+	       }
+	       {
+		    E T4a, T4b, T4d, T4e;
+		    T4a = T3E + T3F;
+		    T4b = T3H + T3I;
+		    T4c = FMA(KP980785280, T4a, KP195090322 * T4b);
+		    T4m = FNMS(KP195090322, T4a, KP980785280 * T4b);
+		    T4d = T3L + T3M;
+		    T4e = T3O + T3P;
+		    T4f = FNMS(KP195090322, T4e, KP980785280 * T4d);
+		    T4n = FMA(KP195090322, T4d, KP980785280 * T4e);
+	       }
+	       {
+		    E T4g, T4q, T4k, T4o;
+		    T4g = T4c + T4f;
+		    ro[WS(os, 17)] = T49 - T4g;
+		    ro[WS(os, 1)] = T49 + T4g;
+		    T4q = T4m + T4n;
+		    io[WS(os, 17)] = T4p - T4q;
+		    io[WS(os, 1)] = T4p + T4q;
+		    T4k = T4f - T4c;
+		    io[WS(os, 25)] = T4j - T4k;
+		    io[WS(os, 9)] = T4j + T4k;
+		    T4o = T4m - T4n;
+		    ro[WS(os, 25)] = T4l - T4o;
+		    ro[WS(os, 9)] = T4l + T4o;
+	       }
+	  }
+	  {
+	       E T3b, T3n, T3l, T3r, T3e, T3o, T3h, T3p;
+	       {
+		    E T39, T3a, T3j, T3k;
+		    T39 = T1z + T1G;
+		    T3a = T2Z + T2Y;
+		    T3b = T39 + T3a;
+		    T3n = T39 - T3a;
+		    T3j = T2T + T2W;
+		    T3k = T1O + T1V;
+		    T3l = T3j - T3k;
+		    T3r = T3j + T3k;
+	       }
+	       {
+		    E T3c, T3d, T3f, T3g;
+		    T3c = T22 + T2d;
+		    T3d = T2j + T2m;
+		    T3e = FMA(KP555570233, T3c, KP831469612 * T3d);
+		    T3o = FNMS(KP555570233, T3d, KP831469612 * T3c);
+		    T3f = T2t + T2E;
+		    T3g = T2K + T2N;
+		    T3h = FNMS(KP555570233, T3g, KP831469612 * T3f);
+		    T3p = FMA(KP831469612, T3g, KP555570233 * T3f);
+	       }
+	       {
+		    E T3i, T3s, T3m, T3q;
+		    T3i = T3e + T3h;
+		    ro[WS(os, 19)] = T3b - T3i;
+		    ro[WS(os, 3)] = T3b + T3i;
+		    T3s = T3o + T3p;
+		    io[WS(os, 19)] = T3r - T3s;
+		    io[WS(os, 3)] = T3r + T3s;
+		    T3m = T3h - T3e;
+		    io[WS(os, 27)] = T3l - T3m;
+		    io[WS(os, 11)] = T3l + T3m;
+		    T3q = T3o - T3p;
+		    ro[WS(os, 27)] = T3n - T3q;
+		    ro[WS(os, 11)] = T3n + T3q;
+	       }
+	  }
+     }
+}
+
+static void m1_32(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; i -= 1) {
+	  m1_32_0(ri, ii, ro, io, is, os);
+	  ri += ivs;
+	  ii += ivs;
+	  ro += ovs;
+	  io += ovs;
+     }
+}
+
+static const kdft_desc desc = { 32, "m1_32", {340, 52, 32, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_m1_32) (planner *p) {
+     X(kdft_register) (p, m1_32, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/m1_64.c b/src/fftw3/dft/codelets/standard/m1_64.c
new file mode 100644
index 0000000..72047bf
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/m1_64.c
@@ -0,0 +1,1568 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:42 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw_noinline -compact -variables 4 -n 64 -name m1_64 -include n.h */
+
+/*
+ * This function contains 912 FP additions, 248 FP multiplications,
+ * (or, 808 additions, 144 multiplications, 104 fused multiply/add),
+ * 171 stack variables, and 256 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: m1_64.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: m1_64.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: m1_64.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void m1_64_0(const R *ri, const R *ii, R *ro, R *io, stride is, stride os)
+{
+     DK(KP773010453, +0.773010453362736960810906609758469800971041293);
+     DK(KP634393284, +0.634393284163645498215171613225493370675687095);
+     DK(KP098017140, +0.098017140329560601994195563888641845861136673);
+     DK(KP995184726, +0.995184726672196886244836953109479921575474869);
+     DK(KP881921264, +0.881921264348355029712756863660388349508442621);
+     DK(KP471396736, +0.471396736825997648556387625905254377657460319);
+     DK(KP290284677, +0.290284677254462367636192375817395274691476278);
+     DK(KP956940335, +0.956940335732208864935797886980269969482849206);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     {
+	  E T37, T7B, T8F, T5Z, Tf, Td9, TbB, TcB, T62, T7C, T2i, TdH, Tah, Tcb, T3e;
+	  E T8G, Tu, TdI, Tak, TbD, Tan, TbC, T2x, Tda, T3m, T65, T7G, T8J, T7J, T8I;
+	  E T3t, T64, TK, Tdd, Tas, Tce, Tav, Tcf, T2N, Tdc, T3G, T6G, T7O, T9k, T7R;
+	  E T9l, T3N, T6H, T1L, Tdv, Tbs, Tcw, TdC, Teo, T5j, T6V, T5Q, T6Y, T8y, T9C;
+	  E Tbb, Tct, T8n, T9z, TZ, Tdf, Taz, Tch, TaC, Tci, T32, Tdg, T3Z, T6J, T7V;
+	  E T9n, T7Y, T9o, T46, T6K, T1g, Tdp, Tb1, Tcm, Tdm, Tej, T4q, T6R, T4X, T6O;
+	  E T8f, T9s, TaK, Tcp, T84, T9v, T1v, Tdn, Tb4, Tcq, Tds, Tek, T4N, T6P, T50;
+	  E T6S, T8i, T9w, TaV, Tcn, T8b, T9t, T20, TdD, Tbv, Tcu, Tdy, Tep, T5G, T6Z;
+	  E T5T, T6W, T8B, T9A, Tbm, Tcx, T8u, T9D;
+	  {
+	       E T3, T35, T26, T5Y, T6, T5X, T29, T36, Ta, T39, T2d, T38, Td, T3b, T2g;
+	       E T3c;
+	       {
+		    E T1, T2, T24, T25;
+		    T1 = ri[0];
+		    T2 = ri[WS(is, 32)];
+		    T3 = T1 + T2;
+		    T35 = T1 - T2;
+		    T24 = ii[0];
+		    T25 = ii[WS(is, 32)];
+		    T26 = T24 + T25;
+		    T5Y = T24 - T25;
+	       }
+	       {
+		    E T4, T5, T27, T28;
+		    T4 = ri[WS(is, 16)];
+		    T5 = ri[WS(is, 48)];
+		    T6 = T4 + T5;
+		    T5X = T4 - T5;
+		    T27 = ii[WS(is, 16)];
+		    T28 = ii[WS(is, 48)];
+		    T29 = T27 + T28;
+		    T36 = T27 - T28;
+	       }
+	       {
+		    E T8, T9, T2b, T2c;
+		    T8 = ri[WS(is, 8)];
+		    T9 = ri[WS(is, 40)];
+		    Ta = T8 + T9;
+		    T39 = T8 - T9;
+		    T2b = ii[WS(is, 8)];
+		    T2c = ii[WS(is, 40)];
+		    T2d = T2b + T2c;
+		    T38 = T2b - T2c;
+	       }
+	       {
+		    E Tb, Tc, T2e, T2f;
+		    Tb = ri[WS(is, 56)];
+		    Tc = ri[WS(is, 24)];
+		    Td = Tb + Tc;
+		    T3b = Tb - Tc;
+		    T2e = ii[WS(is, 56)];
+		    T2f = ii[WS(is, 24)];
+		    T2g = T2e + T2f;
+		    T3c = T2e - T2f;
+	       }
+	       {
+		    E T7, Te, T2a, T2h;
+		    T37 = T35 - T36;
+		    T7B = T35 + T36;
+		    T8F = T5Y - T5X;
+		    T5Z = T5X + T5Y;
+		    T7 = T3 + T6;
+		    Te = Ta + Td;
+		    Tf = T7 + Te;
+		    Td9 = T7 - Te;
+		    {
+			 E Tbz, TbA, T60, T61;
+			 Tbz = T26 - T29;
+			 TbA = Td - Ta;
+			 TbB = Tbz - TbA;
+			 TcB = TbA + Tbz;
+			 T60 = T3b - T3c;
+			 T61 = T39 + T38;
+			 T62 = KP707106781 * (T60 - T61);
+			 T7C = KP707106781 * (T61 + T60);
+		    }
+		    T2a = T26 + T29;
+		    T2h = T2d + T2g;
+		    T2i = T2a + T2h;
+		    TdH = T2a - T2h;
+		    {
+			 E Taf, Tag, T3a, T3d;
+			 Taf = T3 - T6;
+			 Tag = T2d - T2g;
+			 Tah = Taf - Tag;
+			 Tcb = Taf + Tag;
+			 T3a = T38 - T39;
+			 T3d = T3b + T3c;
+			 T3e = KP707106781 * (T3a - T3d);
+			 T8G = KP707106781 * (T3a + T3d);
+		    }
+	       }
+	  }
+	  {
+	       E Ti, T3j, T2l, T3h, Tl, T3g, T2o, T3k, Tp, T3q, T2s, T3o, Ts, T3n, T2v;
+	       E T3r;
+	       {
+		    E Tg, Th, T2j, T2k;
+		    Tg = ri[WS(is, 4)];
+		    Th = ri[WS(is, 36)];
+		    Ti = Tg + Th;
+		    T3j = Tg - Th;
+		    T2j = ii[WS(is, 4)];
+		    T2k = ii[WS(is, 36)];
+		    T2l = T2j + T2k;
+		    T3h = T2j - T2k;
+	       }
+	       {
+		    E Tj, Tk, T2m, T2n;
+		    Tj = ri[WS(is, 20)];
+		    Tk = ri[WS(is, 52)];
+		    Tl = Tj + Tk;
+		    T3g = Tj - Tk;
+		    T2m = ii[WS(is, 20)];
+		    T2n = ii[WS(is, 52)];
+		    T2o = T2m + T2n;
+		    T3k = T2m - T2n;
+	       }
+	       {
+		    E Tn, To, T2q, T2r;
+		    Tn = ri[WS(is, 60)];
+		    To = ri[WS(is, 28)];
+		    Tp = Tn + To;
+		    T3q = Tn - To;
+		    T2q = ii[WS(is, 60)];
+		    T2r = ii[WS(is, 28)];
+		    T2s = T2q + T2r;
+		    T3o = T2q - T2r;
+	       }
+	       {
+		    E Tq, Tr, T2t, T2u;
+		    Tq = ri[WS(is, 12)];
+		    Tr = ri[WS(is, 44)];
+		    Ts = Tq + Tr;
+		    T3n = Tq - Tr;
+		    T2t = ii[WS(is, 12)];
+		    T2u = ii[WS(is, 44)];
+		    T2v = T2t + T2u;
+		    T3r = T2t - T2u;
+	       }
+	       {
+		    E Tm, Tt, Tai, Taj;
+		    Tm = Ti + Tl;
+		    Tt = Tp + Ts;
+		    Tu = Tm + Tt;
+		    TdI = Tt - Tm;
+		    Tai = T2l - T2o;
+		    Taj = Ti - Tl;
+		    Tak = Tai - Taj;
+		    TbD = Taj + Tai;
+	       }
+	       {
+		    E Tal, Tam, T2p, T2w;
+		    Tal = Tp - Ts;
+		    Tam = T2s - T2v;
+		    Tan = Tal + Tam;
+		    TbC = Tal - Tam;
+		    T2p = T2l + T2o;
+		    T2w = T2s + T2v;
+		    T2x = T2p + T2w;
+		    Tda = T2p - T2w;
+	       }
+	       {
+		    E T3i, T3l, T7E, T7F;
+		    T3i = T3g + T3h;
+		    T3l = T3j - T3k;
+		    T3m = FNMS(KP923879532, T3l, KP382683432 * T3i);
+		    T65 = FMA(KP923879532, T3i, KP382683432 * T3l);
+		    T7E = T3h - T3g;
+		    T7F = T3j + T3k;
+		    T7G = FNMS(KP382683432, T7F, KP923879532 * T7E);
+		    T8J = FMA(KP382683432, T7E, KP923879532 * T7F);
+	       }
+	       {
+		    E T7H, T7I, T3p, T3s;
+		    T7H = T3o - T3n;
+		    T7I = T3q + T3r;
+		    T7J = FMA(KP923879532, T7H, KP382683432 * T7I);
+		    T8I = FNMS(KP382683432, T7H, KP923879532 * T7I);
+		    T3p = T3n + T3o;
+		    T3s = T3q - T3r;
+		    T3t = FMA(KP382683432, T3p, KP923879532 * T3s);
+		    T64 = FNMS(KP923879532, T3p, KP382683432 * T3s);
+	       }
+	  }
+	  {
+	       E Ty, T3H, T2B, T3x, TB, T3w, T2E, T3I, TI, T3L, T2L, T3B, TF, T3K, T2I;
+	       E T3E;
+	       {
+		    E Tw, Tx, T2C, T2D;
+		    Tw = ri[WS(is, 2)];
+		    Tx = ri[WS(is, 34)];
+		    Ty = Tw + Tx;
+		    T3H = Tw - Tx;
+		    {
+			 E T2z, T2A, Tz, TA;
+			 T2z = ii[WS(is, 2)];
+			 T2A = ii[WS(is, 34)];
+			 T2B = T2z + T2A;
+			 T3x = T2z - T2A;
+			 Tz = ri[WS(is, 18)];
+			 TA = ri[WS(is, 50)];
+			 TB = Tz + TA;
+			 T3w = Tz - TA;
+		    }
+		    T2C = ii[WS(is, 18)];
+		    T2D = ii[WS(is, 50)];
+		    T2E = T2C + T2D;
+		    T3I = T2C - T2D;
+		    {
+			 E TG, TH, T3z, T2J, T2K, T3A;
+			 TG = ri[WS(is, 58)];
+			 TH = ri[WS(is, 26)];
+			 T3z = TG - TH;
+			 T2J = ii[WS(is, 58)];
+			 T2K = ii[WS(is, 26)];
+			 T3A = T2J - T2K;
+			 TI = TG + TH;
+			 T3L = T3z + T3A;
+			 T2L = T2J + T2K;
+			 T3B = T3z - T3A;
+		    }
+		    {
+			 E TD, TE, T3C, T2G, T2H, T3D;
+			 TD = ri[WS(is, 10)];
+			 TE = ri[WS(is, 42)];
+			 T3C = TD - TE;
+			 T2G = ii[WS(is, 10)];
+			 T2H = ii[WS(is, 42)];
+			 T3D = T2G - T2H;
+			 TF = TD + TE;
+			 T3K = T3D - T3C;
+			 T2I = T2G + T2H;
+			 T3E = T3C + T3D;
+		    }
+	       }
+	       {
+		    E TC, TJ, Taq, Tar;
+		    TC = Ty + TB;
+		    TJ = TF + TI;
+		    TK = TC + TJ;
+		    Tdd = TC - TJ;
+		    Taq = T2B - T2E;
+		    Tar = TI - TF;
+		    Tas = Taq - Tar;
+		    Tce = Tar + Taq;
+	       }
+	       {
+		    E Tat, Tau, T2F, T2M;
+		    Tat = Ty - TB;
+		    Tau = T2I - T2L;
+		    Tav = Tat - Tau;
+		    Tcf = Tat + Tau;
+		    T2F = T2B + T2E;
+		    T2M = T2I + T2L;
+		    T2N = T2F + T2M;
+		    Tdc = T2F - T2M;
+	       }
+	       {
+		    E T3y, T3F, T7M, T7N;
+		    T3y = T3w + T3x;
+		    T3F = KP707106781 * (T3B - T3E);
+		    T3G = T3y - T3F;
+		    T6G = T3y + T3F;
+		    T7M = T3x - T3w;
+		    T7N = KP707106781 * (T3K + T3L);
+		    T7O = T7M - T7N;
+		    T9k = T7M + T7N;
+	       }
+	       {
+		    E T7P, T7Q, T3J, T3M;
+		    T7P = T3H + T3I;
+		    T7Q = KP707106781 * (T3E + T3B);
+		    T7R = T7P - T7Q;
+		    T9l = T7P + T7Q;
+		    T3J = T3H - T3I;
+		    T3M = KP707106781 * (T3K - T3L);
+		    T3N = T3J - T3M;
+		    T6H = T3J + T3M;
+	       }
+	  }
+	  {
+	       E T1z, T53, T5L, Tbo, T1C, T5I, T56, Tbp, T1J, Tb9, T5h, T5N, T1G, Tb8, T5c;
+	       E T5O;
+	       {
+		    E T1x, T1y, T54, T55;
+		    T1x = ri[WS(is, 63)];
+		    T1y = ri[WS(is, 31)];
+		    T1z = T1x + T1y;
+		    T53 = T1x - T1y;
+		    {
+			 E T5J, T5K, T1A, T1B;
+			 T5J = ii[WS(is, 63)];
+			 T5K = ii[WS(is, 31)];
+			 T5L = T5J - T5K;
+			 Tbo = T5J + T5K;
+			 T1A = ri[WS(is, 15)];
+			 T1B = ri[WS(is, 47)];
+			 T1C = T1A + T1B;
+			 T5I = T1A - T1B;
+		    }
+		    T54 = ii[WS(is, 15)];
+		    T55 = ii[WS(is, 47)];
+		    T56 = T54 - T55;
+		    Tbp = T54 + T55;
+		    {
+			 E T1H, T1I, T5d, T5e, T5f, T5g;
+			 T1H = ri[WS(is, 55)];
+			 T1I = ri[WS(is, 23)];
+			 T5d = T1H - T1I;
+			 T5e = ii[WS(is, 55)];
+			 T5f = ii[WS(is, 23)];
+			 T5g = T5e - T5f;
+			 T1J = T1H + T1I;
+			 Tb9 = T5e + T5f;
+			 T5h = T5d + T5g;
+			 T5N = T5d - T5g;
+		    }
+		    {
+			 E T1E, T1F, T5b, T58, T59, T5a;
+			 T1E = ri[WS(is, 7)];
+			 T1F = ri[WS(is, 39)];
+			 T5b = T1E - T1F;
+			 T58 = ii[WS(is, 7)];
+			 T59 = ii[WS(is, 39)];
+			 T5a = T58 - T59;
+			 T1G = T1E + T1F;
+			 Tb8 = T58 + T59;
+			 T5c = T5a - T5b;
+			 T5O = T5b + T5a;
+		    }
+	       }
+	       {
+		    E T1D, T1K, Tbq, Tbr;
+		    T1D = T1z + T1C;
+		    T1K = T1G + T1J;
+		    T1L = T1D + T1K;
+		    Tdv = T1D - T1K;
+		    Tbq = Tbo - Tbp;
+		    Tbr = T1J - T1G;
+		    Tbs = Tbq - Tbr;
+		    Tcw = Tbr + Tbq;
+	       }
+	       {
+		    E TdA, TdB, T57, T5i;
+		    TdA = Tbo + Tbp;
+		    TdB = Tb8 + Tb9;
+		    TdC = TdA - TdB;
+		    Teo = TdA + TdB;
+		    T57 = T53 - T56;
+		    T5i = KP707106781 * (T5c - T5h);
+		    T5j = T57 - T5i;
+		    T6V = T57 + T5i;
+	       }
+	       {
+		    E T5M, T5P, T8w, T8x;
+		    T5M = T5I + T5L;
+		    T5P = KP707106781 * (T5N - T5O);
+		    T5Q = T5M - T5P;
+		    T6Y = T5M + T5P;
+		    T8w = T5L - T5I;
+		    T8x = KP707106781 * (T5c + T5h);
+		    T8y = T8w - T8x;
+		    T9C = T8w + T8x;
+	       }
+	       {
+		    E Tb7, Tba, T8l, T8m;
+		    Tb7 = T1z - T1C;
+		    Tba = Tb8 - Tb9;
+		    Tbb = Tb7 - Tba;
+		    Tct = Tb7 + Tba;
+		    T8l = T53 + T56;
+		    T8m = KP707106781 * (T5O + T5N);
+		    T8n = T8l - T8m;
+		    T9z = T8l + T8m;
+	       }
+	  }
+	  {
+	       E TN, T40, T2Q, T3Q, TQ, T3P, T2T, T41, TX, T44, T30, T3U, TU, T43, T2X;
+	       E T3X;
+	       {
+		    E TL, TM, T2R, T2S;
+		    TL = ri[WS(is, 62)];
+		    TM = ri[WS(is, 30)];
+		    TN = TL + TM;
+		    T40 = TL - TM;
+		    {
+			 E T2O, T2P, TO, TP;
+			 T2O = ii[WS(is, 62)];
+			 T2P = ii[WS(is, 30)];
+			 T2Q = T2O + T2P;
+			 T3Q = T2O - T2P;
+			 TO = ri[WS(is, 14)];
+			 TP = ri[WS(is, 46)];
+			 TQ = TO + TP;
+			 T3P = TO - TP;
+		    }
+		    T2R = ii[WS(is, 14)];
+		    T2S = ii[WS(is, 46)];
+		    T2T = T2R + T2S;
+		    T41 = T2R - T2S;
+		    {
+			 E TV, TW, T3S, T2Y, T2Z, T3T;
+			 TV = ri[WS(is, 54)];
+			 TW = ri[WS(is, 22)];
+			 T3S = TV - TW;
+			 T2Y = ii[WS(is, 54)];
+			 T2Z = ii[WS(is, 22)];
+			 T3T = T2Y - T2Z;
+			 TX = TV + TW;
+			 T44 = T3S + T3T;
+			 T30 = T2Y + T2Z;
+			 T3U = T3S - T3T;
+		    }
+		    {
+			 E TS, TT, T3V, T2V, T2W, T3W;
+			 TS = ri[WS(is, 6)];
+			 TT = ri[WS(is, 38)];
+			 T3V = TS - TT;
+			 T2V = ii[WS(is, 6)];
+			 T2W = ii[WS(is, 38)];
+			 T3W = T2V - T2W;
+			 TU = TS + TT;
+			 T43 = T3W - T3V;
+			 T2X = T2V + T2W;
+			 T3X = T3V + T3W;
+		    }
+	       }
+	       {
+		    E TR, TY, Tax, Tay;
+		    TR = TN + TQ;
+		    TY = TU + TX;
+		    TZ = TR + TY;
+		    Tdf = TR - TY;
+		    Tax = T2Q - T2T;
+		    Tay = TX - TU;
+		    Taz = Tax - Tay;
+		    Tch = Tay + Tax;
+	       }
+	       {
+		    E TaA, TaB, T2U, T31;
+		    TaA = TN - TQ;
+		    TaB = T2X - T30;
+		    TaC = TaA - TaB;
+		    Tci = TaA + TaB;
+		    T2U = T2Q + T2T;
+		    T31 = T2X + T30;
+		    T32 = T2U + T31;
+		    Tdg = T2U - T31;
+	       }
+	       {
+		    E T3R, T3Y, T7T, T7U;
+		    T3R = T3P + T3Q;
+		    T3Y = KP707106781 * (T3U - T3X);
+		    T3Z = T3R - T3Y;
+		    T6J = T3R + T3Y;
+		    T7T = T40 + T41;
+		    T7U = KP707106781 * (T3X + T3U);
+		    T7V = T7T - T7U;
+		    T9n = T7T + T7U;
+	       }
+	       {
+		    E T7W, T7X, T42, T45;
+		    T7W = T3Q - T3P;
+		    T7X = KP707106781 * (T43 + T44);
+		    T7Y = T7W - T7X;
+		    T9o = T7W + T7X;
+		    T42 = T40 - T41;
+		    T45 = KP707106781 * (T43 - T44);
+		    T46 = T42 - T45;
+		    T6K = T42 + T45;
+	       }
+	  }
+	  {
+	       E T14, T4P, T4d, TaG, T17, T4a, T4S, TaH, T1e, TaZ, T4j, T4V, T1b, TaY, T4o;
+	       E T4U;
+	       {
+		    E T12, T13, T4Q, T4R;
+		    T12 = ri[WS(is, 1)];
+		    T13 = ri[WS(is, 33)];
+		    T14 = T12 + T13;
+		    T4P = T12 - T13;
+		    {
+			 E T4b, T4c, T15, T16;
+			 T4b = ii[WS(is, 1)];
+			 T4c = ii[WS(is, 33)];
+			 T4d = T4b - T4c;
+			 TaG = T4b + T4c;
+			 T15 = ri[WS(is, 17)];
+			 T16 = ri[WS(is, 49)];
+			 T17 = T15 + T16;
+			 T4a = T15 - T16;
+		    }
+		    T4Q = ii[WS(is, 17)];
+		    T4R = ii[WS(is, 49)];
+		    T4S = T4Q - T4R;
+		    TaH = T4Q + T4R;
+		    {
+			 E T1c, T1d, T4f, T4g, T4h, T4i;
+			 T1c = ri[WS(is, 57)];
+			 T1d = ri[WS(is, 25)];
+			 T4f = T1c - T1d;
+			 T4g = ii[WS(is, 57)];
+			 T4h = ii[WS(is, 25)];
+			 T4i = T4g - T4h;
+			 T1e = T1c + T1d;
+			 TaZ = T4g + T4h;
+			 T4j = T4f - T4i;
+			 T4V = T4f + T4i;
+		    }
+		    {
+			 E T19, T1a, T4k, T4l, T4m, T4n;
+			 T19 = ri[WS(is, 9)];
+			 T1a = ri[WS(is, 41)];
+			 T4k = T19 - T1a;
+			 T4l = ii[WS(is, 9)];
+			 T4m = ii[WS(is, 41)];
+			 T4n = T4l - T4m;
+			 T1b = T19 + T1a;
+			 TaY = T4l + T4m;
+			 T4o = T4k + T4n;
+			 T4U = T4n - T4k;
+		    }
+	       }
+	       {
+		    E T18, T1f, TaX, Tb0;
+		    T18 = T14 + T17;
+		    T1f = T1b + T1e;
+		    T1g = T18 + T1f;
+		    Tdp = T18 - T1f;
+		    TaX = T14 - T17;
+		    Tb0 = TaY - TaZ;
+		    Tb1 = TaX - Tb0;
+		    Tcm = TaX + Tb0;
+	       }
+	       {
+		    E Tdk, Tdl, T4e, T4p;
+		    Tdk = TaG + TaH;
+		    Tdl = TaY + TaZ;
+		    Tdm = Tdk - Tdl;
+		    Tej = Tdk + Tdl;
+		    T4e = T4a + T4d;
+		    T4p = KP707106781 * (T4j - T4o);
+		    T4q = T4e - T4p;
+		    T6R = T4e + T4p;
+	       }
+	       {
+		    E T4T, T4W, T8d, T8e;
+		    T4T = T4P - T4S;
+		    T4W = KP707106781 * (T4U - T4V);
+		    T4X = T4T - T4W;
+		    T6O = T4T + T4W;
+		    T8d = T4P + T4S;
+		    T8e = KP707106781 * (T4o + T4j);
+		    T8f = T8d - T8e;
+		    T9s = T8d + T8e;
+	       }
+	       {
+		    E TaI, TaJ, T82, T83;
+		    TaI = TaG - TaH;
+		    TaJ = T1e - T1b;
+		    TaK = TaI - TaJ;
+		    Tcp = TaJ + TaI;
+		    T82 = T4d - T4a;
+		    T83 = KP707106781 * (T4U + T4V);
+		    T84 = T82 - T83;
+		    T9v = T82 + T83;
+	       }
+	  }
+	  {
+	       E T1j, TaR, T1m, TaS, T4G, T4L, TaT, TaQ, T89, T88, T1q, TaM, T1t, TaN, T4v;
+	       E T4A, TaO, TaL, T86, T85;
+	       {
+		    E T4H, T4F, T4C, T4K;
+		    {
+			 E T1h, T1i, T4D, T4E;
+			 T1h = ri[WS(is, 5)];
+			 T1i = ri[WS(is, 37)];
+			 T1j = T1h + T1i;
+			 T4H = T1h - T1i;
+			 T4D = ii[WS(is, 5)];
+			 T4E = ii[WS(is, 37)];
+			 T4F = T4D - T4E;
+			 TaR = T4D + T4E;
+		    }
+		    {
+			 E T1k, T1l, T4I, T4J;
+			 T1k = ri[WS(is, 21)];
+			 T1l = ri[WS(is, 53)];
+			 T1m = T1k + T1l;
+			 T4C = T1k - T1l;
+			 T4I = ii[WS(is, 21)];
+			 T4J = ii[WS(is, 53)];
+			 T4K = T4I - T4J;
+			 TaS = T4I + T4J;
+		    }
+		    T4G = T4C + T4F;
+		    T4L = T4H - T4K;
+		    TaT = TaR - TaS;
+		    TaQ = T1j - T1m;
+		    T89 = T4H + T4K;
+		    T88 = T4F - T4C;
+	       }
+	       {
+		    E T4r, T4z, T4w, T4u;
+		    {
+			 E T1o, T1p, T4x, T4y;
+			 T1o = ri[WS(is, 61)];
+			 T1p = ri[WS(is, 29)];
+			 T1q = T1o + T1p;
+			 T4r = T1o - T1p;
+			 T4x = ii[WS(is, 61)];
+			 T4y = ii[WS(is, 29)];
+			 T4z = T4x - T4y;
+			 TaM = T4x + T4y;
+		    }
+		    {
+			 E T1r, T1s, T4s, T4t;
+			 T1r = ri[WS(is, 13)];
+			 T1s = ri[WS(is, 45)];
+			 T1t = T1r + T1s;
+			 T4w = T1r - T1s;
+			 T4s = ii[WS(is, 13)];
+			 T4t = ii[WS(is, 45)];
+			 T4u = T4s - T4t;
+			 TaN = T4s + T4t;
+		    }
+		    T4v = T4r - T4u;
+		    T4A = T4w + T4z;
+		    TaO = TaM - TaN;
+		    TaL = T1q - T1t;
+		    T86 = T4z - T4w;
+		    T85 = T4r + T4u;
+	       }
+	       {
+		    E T1n, T1u, Tb2, Tb3;
+		    T1n = T1j + T1m;
+		    T1u = T1q + T1t;
+		    T1v = T1n + T1u;
+		    Tdn = T1u - T1n;
+		    Tb2 = TaT - TaQ;
+		    Tb3 = TaL + TaO;
+		    Tb4 = KP707106781 * (Tb2 - Tb3);
+		    Tcq = KP707106781 * (Tb2 + Tb3);
+	       }
+	       {
+		    E Tdq, Tdr, T4B, T4M;
+		    Tdq = TaR + TaS;
+		    Tdr = TaM + TaN;
+		    Tds = Tdq - Tdr;
+		    Tek = Tdq + Tdr;
+		    T4B = FNMS(KP923879532, T4A, KP382683432 * T4v);
+		    T4M = FMA(KP923879532, T4G, KP382683432 * T4L);
+		    T4N = T4B - T4M;
+		    T6P = T4M + T4B;
+	       }
+	       {
+		    E T4Y, T4Z, T8g, T8h;
+		    T4Y = FNMS(KP923879532, T4L, KP382683432 * T4G);
+		    T4Z = FMA(KP382683432, T4A, KP923879532 * T4v);
+		    T50 = T4Y - T4Z;
+		    T6S = T4Y + T4Z;
+		    T8g = FNMS(KP382683432, T89, KP923879532 * T88);
+		    T8h = FMA(KP923879532, T86, KP382683432 * T85);
+		    T8i = T8g - T8h;
+		    T9w = T8g + T8h;
+	       }
+	       {
+		    E TaP, TaU, T87, T8a;
+		    TaP = TaL - TaO;
+		    TaU = TaQ + TaT;
+		    TaV = KP707106781 * (TaP - TaU);
+		    Tcn = KP707106781 * (TaU + TaP);
+		    T87 = FNMS(KP382683432, T86, KP923879532 * T85);
+		    T8a = FMA(KP382683432, T88, KP923879532 * T89);
+		    T8b = T87 - T8a;
+		    T9t = T8a + T87;
+	       }
+	  }
+	  {
+	       E T1O, Tbc, T1R, Tbd, T5o, T5t, Tbf, Tbe, T8p, T8o, T1V, Tbi, T1Y, Tbj, T5z;
+	       E T5E, Tbk, Tbh, T8s, T8r;
+	       {
+		    E T5p, T5n, T5k, T5s;
+		    {
+			 E T1M, T1N, T5l, T5m;
+			 T1M = ri[WS(is, 3)];
+			 T1N = ri[WS(is, 35)];
+			 T1O = T1M + T1N;
+			 T5p = T1M - T1N;
+			 T5l = ii[WS(is, 3)];
+			 T5m = ii[WS(is, 35)];
+			 T5n = T5l - T5m;
+			 Tbc = T5l + T5m;
+		    }
+		    {
+			 E T1P, T1Q, T5q, T5r;
+			 T1P = ri[WS(is, 19)];
+			 T1Q = ri[WS(is, 51)];
+			 T1R = T1P + T1Q;
+			 T5k = T1P - T1Q;
+			 T5q = ii[WS(is, 19)];
+			 T5r = ii[WS(is, 51)];
+			 T5s = T5q - T5r;
+			 Tbd = T5q + T5r;
+		    }
+		    T5o = T5k + T5n;
+		    T5t = T5p - T5s;
+		    Tbf = T1O - T1R;
+		    Tbe = Tbc - Tbd;
+		    T8p = T5p + T5s;
+		    T8o = T5n - T5k;
+	       }
+	       {
+		    E T5A, T5y, T5v, T5D;
+		    {
+			 E T1T, T1U, T5w, T5x;
+			 T1T = ri[WS(is, 59)];
+			 T1U = ri[WS(is, 27)];
+			 T1V = T1T + T1U;
+			 T5A = T1T - T1U;
+			 T5w = ii[WS(is, 59)];
+			 T5x = ii[WS(is, 27)];
+			 T5y = T5w - T5x;
+			 Tbi = T5w + T5x;
+		    }
+		    {
+			 E T1W, T1X, T5B, T5C;
+			 T1W = ri[WS(is, 11)];
+			 T1X = ri[WS(is, 43)];
+			 T1Y = T1W + T1X;
+			 T5v = T1W - T1X;
+			 T5B = ii[WS(is, 11)];
+			 T5C = ii[WS(is, 43)];
+			 T5D = T5B - T5C;
+			 Tbj = T5B + T5C;
+		    }
+		    T5z = T5v + T5y;
+		    T5E = T5A - T5D;
+		    Tbk = Tbi - Tbj;
+		    Tbh = T1V - T1Y;
+		    T8s = T5A + T5D;
+		    T8r = T5y - T5v;
+	       }
+	       {
+		    E T1S, T1Z, Tbt, Tbu;
+		    T1S = T1O + T1R;
+		    T1Z = T1V + T1Y;
+		    T20 = T1S + T1Z;
+		    TdD = T1Z - T1S;
+		    Tbt = Tbh - Tbk;
+		    Tbu = Tbf + Tbe;
+		    Tbv = KP707106781 * (Tbt - Tbu);
+		    Tcu = KP707106781 * (Tbu + Tbt);
+	       }
+	       {
+		    E Tdw, Tdx, T5u, T5F;
+		    Tdw = Tbc + Tbd;
+		    Tdx = Tbi + Tbj;
+		    Tdy = Tdw - Tdx;
+		    Tep = Tdw + Tdx;
+		    T5u = FNMS(KP923879532, T5t, KP382683432 * T5o);
+		    T5F = FMA(KP382683432, T5z, KP923879532 * T5E);
+		    T5G = T5u - T5F;
+		    T6Z = T5u + T5F;
+	       }
+	       {
+		    E T5R, T5S, T8z, T8A;
+		    T5R = FNMS(KP923879532, T5z, KP382683432 * T5E);
+		    T5S = FMA(KP923879532, T5o, KP382683432 * T5t);
+		    T5T = T5R - T5S;
+		    T6W = T5S + T5R;
+		    T8z = FNMS(KP382683432, T8r, KP923879532 * T8s);
+		    T8A = FMA(KP382683432, T8o, KP923879532 * T8p);
+		    T8B = T8z - T8A;
+		    T9A = T8A + T8z;
+	       }
+	       {
+		    E Tbg, Tbl, T8q, T8t;
+		    Tbg = Tbe - Tbf;
+		    Tbl = Tbh + Tbk;
+		    Tbm = KP707106781 * (Tbg - Tbl);
+		    Tcx = KP707106781 * (Tbg + Tbl);
+		    T8q = FNMS(KP382683432, T8p, KP923879532 * T8o);
+		    T8t = FMA(KP923879532, T8r, KP382683432 * T8s);
+		    T8u = T8q - T8t;
+		    T9D = T8q + T8t;
+	       }
+	  }
+	  {
+	       E T11, TeD, TeG, TeI, T22, T23, T34, TeH;
+	       {
+		    E Tv, T10, TeE, TeF;
+		    Tv = Tf + Tu;
+		    T10 = TK + TZ;
+		    T11 = Tv + T10;
+		    TeD = Tv - T10;
+		    TeE = Tej + Tek;
+		    TeF = Teo + Tep;
+		    TeG = TeE - TeF;
+		    TeI = TeE + TeF;
+	       }
+	       {
+		    E T1w, T21, T2y, T33;
+		    T1w = T1g + T1v;
+		    T21 = T1L + T20;
+		    T22 = T1w + T21;
+		    T23 = T21 - T1w;
+		    T2y = T2i + T2x;
+		    T33 = T2N + T32;
+		    T34 = T2y - T33;
+		    TeH = T2y + T33;
+	       }
+	       ro[WS(os, 32)] = T11 - T22;
+	       io[WS(os, 32)] = TeH - TeI;
+	       ro[0] = T11 + T22;
+	       io[0] = TeH + TeI;
+	       io[WS(os, 16)] = T23 + T34;
+	       ro[WS(os, 16)] = TeD + TeG;
+	       io[WS(os, 48)] = T34 - T23;
+	       ro[WS(os, 48)] = TeD - TeG;
+	  }
+	  {
+	       E Teh, Tex, Tev, TeB, Tem, Tey, Ter, Tez;
+	       {
+		    E Tef, Teg, Tet, Teu;
+		    Tef = Tf - Tu;
+		    Teg = T2N - T32;
+		    Teh = Tef + Teg;
+		    Tex = Tef - Teg;
+		    Tet = T2i - T2x;
+		    Teu = TZ - TK;
+		    Tev = Tet - Teu;
+		    TeB = Teu + Tet;
+	       }
+	       {
+		    E Tei, Tel, Ten, Teq;
+		    Tei = T1g - T1v;
+		    Tel = Tej - Tek;
+		    Tem = Tei + Tel;
+		    Tey = Tel - Tei;
+		    Ten = T1L - T20;
+		    Teq = Teo - Tep;
+		    Ter = Ten - Teq;
+		    Tez = Ten + Teq;
+	       }
+	       {
+		    E Tes, TeC, Tew, TeA;
+		    Tes = KP707106781 * (Tem + Ter);
+		    ro[WS(os, 40)] = Teh - Tes;
+		    ro[WS(os, 8)] = Teh + Tes;
+		    TeC = KP707106781 * (Tey + Tez);
+		    io[WS(os, 40)] = TeB - TeC;
+		    io[WS(os, 8)] = TeB + TeC;
+		    Tew = KP707106781 * (Ter - Tem);
+		    io[WS(os, 56)] = Tev - Tew;
+		    io[WS(os, 24)] = Tev + Tew;
+		    TeA = KP707106781 * (Tey - Tez);
+		    ro[WS(os, 56)] = Tex - TeA;
+		    ro[WS(os, 24)] = Tex + TeA;
+	       }
+	  }
+	  {
+	       E Tdb, TdV, Te5, TdJ, Tdi, Te6, Te3, Teb, TdM, TdW, Tdu, TdQ, Te0, Tea, TdF;
+	       E TdR;
+	       {
+		    E Tde, Tdh, Tdo, Tdt;
+		    Tdb = Td9 - Tda;
+		    TdV = Td9 + Tda;
+		    Te5 = TdI + TdH;
+		    TdJ = TdH - TdI;
+		    Tde = Tdc - Tdd;
+		    Tdh = Tdf + Tdg;
+		    Tdi = KP707106781 * (Tde - Tdh);
+		    Te6 = KP707106781 * (Tde + Tdh);
+		    {
+			 E Te1, Te2, TdK, TdL;
+			 Te1 = Tdv + Tdy;
+			 Te2 = TdD + TdC;
+			 Te3 = FNMS(KP382683432, Te2, KP923879532 * Te1);
+			 Teb = FMA(KP923879532, Te2, KP382683432 * Te1);
+			 TdK = Tdf - Tdg;
+			 TdL = Tdd + Tdc;
+			 TdM = KP707106781 * (TdK - TdL);
+			 TdW = KP707106781 * (TdL + TdK);
+		    }
+		    Tdo = Tdm - Tdn;
+		    Tdt = Tdp - Tds;
+		    Tdu = FMA(KP923879532, Tdo, KP382683432 * Tdt);
+		    TdQ = FNMS(KP923879532, Tdt, KP382683432 * Tdo);
+		    {
+			 E TdY, TdZ, Tdz, TdE;
+			 TdY = Tdn + Tdm;
+			 TdZ = Tdp + Tds;
+			 Te0 = FMA(KP382683432, TdY, KP923879532 * TdZ);
+			 Tea = FNMS(KP382683432, TdZ, KP923879532 * TdY);
+			 Tdz = Tdv - Tdy;
+			 TdE = TdC - TdD;
+			 TdF = FNMS(KP923879532, TdE, KP382683432 * Tdz);
+			 TdR = FMA(KP382683432, TdE, KP923879532 * Tdz);
+		    }
+	       }
+	       {
+		    E Tdj, TdG, TdT, TdU;
+		    Tdj = Tdb + Tdi;
+		    TdG = Tdu + TdF;
+		    ro[WS(os, 44)] = Tdj - TdG;
+		    ro[WS(os, 12)] = Tdj + TdG;
+		    TdT = TdJ + TdM;
+		    TdU = TdQ + TdR;
+		    io[WS(os, 44)] = TdT - TdU;
+		    io[WS(os, 12)] = TdT + TdU;
+	       }
+	       {
+		    E TdN, TdO, TdP, TdS;
+		    TdN = TdJ - TdM;
+		    TdO = TdF - Tdu;
+		    io[WS(os, 60)] = TdN - TdO;
+		    io[WS(os, 28)] = TdN + TdO;
+		    TdP = Tdb - Tdi;
+		    TdS = TdQ - TdR;
+		    ro[WS(os, 60)] = TdP - TdS;
+		    ro[WS(os, 28)] = TdP + TdS;
+	       }
+	       {
+		    E TdX, Te4, Ted, Tee;
+		    TdX = TdV + TdW;
+		    Te4 = Te0 + Te3;
+		    ro[WS(os, 36)] = TdX - Te4;
+		    ro[WS(os, 4)] = TdX + Te4;
+		    Ted = Te5 + Te6;
+		    Tee = Tea + Teb;
+		    io[WS(os, 36)] = Ted - Tee;
+		    io[WS(os, 4)] = Ted + Tee;
+	       }
+	       {
+		    E Te7, Te8, Te9, Tec;
+		    Te7 = Te5 - Te6;
+		    Te8 = Te3 - Te0;
+		    io[WS(os, 52)] = Te7 - Te8;
+		    io[WS(os, 20)] = Te7 + Te8;
+		    Te9 = TdV - TdW;
+		    Tec = Tea - Teb;
+		    ro[WS(os, 52)] = Te9 - Tec;
+		    ro[WS(os, 20)] = Te9 + Tec;
+	       }
+	  }
+	  {
+	       E Tcd, TcP, TcD, TcZ, Tck, Td0, TcX, Td5, Tcs, TcK, TcG, TcQ, TcU, Td4, Tcz;
+	       E TcL, Tcc, TcC;
+	       Tcc = KP707106781 * (TbD + TbC);
+	       Tcd = Tcb - Tcc;
+	       TcP = Tcb + Tcc;
+	       TcC = KP707106781 * (Tak + Tan);
+	       TcD = TcB - TcC;
+	       TcZ = TcB + TcC;
+	       {
+		    E Tcg, Tcj, TcV, TcW;
+		    Tcg = FNMS(KP382683432, Tcf, KP923879532 * Tce);
+		    Tcj = FMA(KP923879532, Tch, KP382683432 * Tci);
+		    Tck = Tcg - Tcj;
+		    Td0 = Tcg + Tcj;
+		    TcV = Tct + Tcu;
+		    TcW = Tcw + Tcx;
+		    TcX = FNMS(KP195090322, TcW, KP980785280 * TcV);
+		    Td5 = FMA(KP195090322, TcV, KP980785280 * TcW);
+	       }
+	       {
+		    E Tco, Tcr, TcE, TcF;
+		    Tco = Tcm - Tcn;
+		    Tcr = Tcp - Tcq;
+		    Tcs = FMA(KP555570233, Tco, KP831469612 * Tcr);
+		    TcK = FNMS(KP831469612, Tco, KP555570233 * Tcr);
+		    TcE = FNMS(KP382683432, Tch, KP923879532 * Tci);
+		    TcF = FMA(KP382683432, Tce, KP923879532 * Tcf);
+		    TcG = TcE - TcF;
+		    TcQ = TcF + TcE;
+	       }
+	       {
+		    E TcS, TcT, Tcv, Tcy;
+		    TcS = Tcm + Tcn;
+		    TcT = Tcp + Tcq;
+		    TcU = FMA(KP980785280, TcS, KP195090322 * TcT);
+		    Td4 = FNMS(KP195090322, TcS, KP980785280 * TcT);
+		    Tcv = Tct - Tcu;
+		    Tcy = Tcw - Tcx;
+		    Tcz = FNMS(KP831469612, Tcy, KP555570233 * Tcv);
+		    TcL = FMA(KP831469612, Tcv, KP555570233 * Tcy);
+	       }
+	       {
+		    E Tcl, TcA, TcN, TcO;
+		    Tcl = Tcd + Tck;
+		    TcA = Tcs + Tcz;
+		    ro[WS(os, 42)] = Tcl - TcA;
+		    ro[WS(os, 10)] = Tcl + TcA;
+		    TcN = TcD + TcG;
+		    TcO = TcK + TcL;
+		    io[WS(os, 42)] = TcN - TcO;
+		    io[WS(os, 10)] = TcN + TcO;
+	       }
+	       {
+		    E TcH, TcI, TcJ, TcM;
+		    TcH = TcD - TcG;
+		    TcI = Tcz - Tcs;
+		    io[WS(os, 58)] = TcH - TcI;
+		    io[WS(os, 26)] = TcH + TcI;
+		    TcJ = Tcd - Tck;
+		    TcM = TcK - TcL;
+		    ro[WS(os, 58)] = TcJ - TcM;
+		    ro[WS(os, 26)] = TcJ + TcM;
+	       }
+	       {
+		    E TcR, TcY, Td7, Td8;
+		    TcR = TcP + TcQ;
+		    TcY = TcU + TcX;
+		    ro[WS(os, 34)] = TcR - TcY;
+		    ro[WS(os, 2)] = TcR + TcY;
+		    Td7 = TcZ + Td0;
+		    Td8 = Td4 + Td5;
+		    io[WS(os, 34)] = Td7 - Td8;
+		    io[WS(os, 2)] = Td7 + Td8;
+	       }
+	       {
+		    E Td1, Td2, Td3, Td6;
+		    Td1 = TcZ - Td0;
+		    Td2 = TcX - TcU;
+		    io[WS(os, 50)] = Td1 - Td2;
+		    io[WS(os, 18)] = Td1 + Td2;
+		    Td3 = TcP - TcQ;
+		    Td6 = Td4 - Td5;
+		    ro[WS(os, 50)] = Td3 - Td6;
+		    ro[WS(os, 18)] = Td3 + Td6;
+	       }
+	  }
+	  {
+	       E Tap, TbR, TbF, Tc1, TaE, Tc2, TbZ, Tc7, Tb6, TbM, TbI, TbS, TbW, Tc6, Tbx;
+	       E TbN, Tao, TbE;
+	       Tao = KP707106781 * (Tak - Tan);
+	       Tap = Tah - Tao;
+	       TbR = Tah + Tao;
+	       TbE = KP707106781 * (TbC - TbD);
+	       TbF = TbB - TbE;
+	       Tc1 = TbB + TbE;
+	       {
+		    E Taw, TaD, TbX, TbY;
+		    Taw = FNMS(KP923879532, Tav, KP382683432 * Tas);
+		    TaD = FMA(KP382683432, Taz, KP923879532 * TaC);
+		    TaE = Taw - TaD;
+		    Tc2 = Taw + TaD;
+		    TbX = Tbb + Tbm;
+		    TbY = Tbs + Tbv;
+		    TbZ = FNMS(KP555570233, TbY, KP831469612 * TbX);
+		    Tc7 = FMA(KP831469612, TbY, KP555570233 * TbX);
+	       }
+	       {
+		    E TaW, Tb5, TbG, TbH;
+		    TaW = TaK - TaV;
+		    Tb5 = Tb1 - Tb4;
+		    Tb6 = FMA(KP980785280, TaW, KP195090322 * Tb5);
+		    TbM = FNMS(KP980785280, Tb5, KP195090322 * TaW);
+		    TbG = FNMS(KP923879532, Taz, KP382683432 * TaC);
+		    TbH = FMA(KP923879532, Tas, KP382683432 * Tav);
+		    TbI = TbG - TbH;
+		    TbS = TbH + TbG;
+	       }
+	       {
+		    E TbU, TbV, Tbn, Tbw;
+		    TbU = TaK + TaV;
+		    TbV = Tb1 + Tb4;
+		    TbW = FMA(KP555570233, TbU, KP831469612 * TbV);
+		    Tc6 = FNMS(KP555570233, TbV, KP831469612 * TbU);
+		    Tbn = Tbb - Tbm;
+		    Tbw = Tbs - Tbv;
+		    Tbx = FNMS(KP980785280, Tbw, KP195090322 * Tbn);
+		    TbN = FMA(KP195090322, Tbw, KP980785280 * Tbn);
+	       }
+	       {
+		    E TaF, Tby, TbP, TbQ;
+		    TaF = Tap + TaE;
+		    Tby = Tb6 + Tbx;
+		    ro[WS(os, 46)] = TaF - Tby;
+		    ro[WS(os, 14)] = TaF + Tby;
+		    TbP = TbF + TbI;
+		    TbQ = TbM + TbN;
+		    io[WS(os, 46)] = TbP - TbQ;
+		    io[WS(os, 14)] = TbP + TbQ;
+	       }
+	       {
+		    E TbJ, TbK, TbL, TbO;
+		    TbJ = TbF - TbI;
+		    TbK = Tbx - Tb6;
+		    io[WS(os, 62)] = TbJ - TbK;
+		    io[WS(os, 30)] = TbJ + TbK;
+		    TbL = Tap - TaE;
+		    TbO = TbM - TbN;
+		    ro[WS(os, 62)] = TbL - TbO;
+		    ro[WS(os, 30)] = TbL + TbO;
+	       }
+	       {
+		    E TbT, Tc0, Tc9, Tca;
+		    TbT = TbR + TbS;
+		    Tc0 = TbW + TbZ;
+		    ro[WS(os, 38)] = TbT - Tc0;
+		    ro[WS(os, 6)] = TbT + Tc0;
+		    Tc9 = Tc1 + Tc2;
+		    Tca = Tc6 + Tc7;
+		    io[WS(os, 38)] = Tc9 - Tca;
+		    io[WS(os, 6)] = Tc9 + Tca;
+	       }
+	       {
+		    E Tc3, Tc4, Tc5, Tc8;
+		    Tc3 = Tc1 - Tc2;
+		    Tc4 = TbZ - TbW;
+		    io[WS(os, 54)] = Tc3 - Tc4;
+		    io[WS(os, 22)] = Tc3 + Tc4;
+		    Tc5 = TbR - TbS;
+		    Tc8 = Tc6 - Tc7;
+		    ro[WS(os, 54)] = Tc5 - Tc8;
+		    ro[WS(os, 22)] = Tc5 + Tc8;
+	       }
+	  }
+	  {
+	       E T6F, T7h, T7m, T7w, T7p, T7x, T6M, T7s, T6U, T7c, T75, T7r, T78, T7i, T71;
+	       E T7d;
+	       {
+		    E T6D, T6E, T7k, T7l;
+		    T6D = T37 + T3e;
+		    T6E = T65 + T64;
+		    T6F = T6D - T6E;
+		    T7h = T6D + T6E;
+		    T7k = T6O + T6P;
+		    T7l = T6R + T6S;
+		    T7m = FMA(KP956940335, T7k, KP290284677 * T7l);
+		    T7w = FNMS(KP290284677, T7k, KP956940335 * T7l);
+	       }
+	       {
+		    E T7n, T7o, T6I, T6L;
+		    T7n = T6V + T6W;
+		    T7o = T6Y + T6Z;
+		    T7p = FNMS(KP290284677, T7o, KP956940335 * T7n);
+		    T7x = FMA(KP290284677, T7n, KP956940335 * T7o);
+		    T6I = FNMS(KP555570233, T6H, KP831469612 * T6G);
+		    T6L = FMA(KP831469612, T6J, KP555570233 * T6K);
+		    T6M = T6I - T6L;
+		    T7s = T6I + T6L;
+	       }
+	       {
+		    E T6Q, T6T, T73, T74;
+		    T6Q = T6O - T6P;
+		    T6T = T6R - T6S;
+		    T6U = FMA(KP471396736, T6Q, KP881921264 * T6T);
+		    T7c = FNMS(KP881921264, T6Q, KP471396736 * T6T);
+		    T73 = T5Z + T62;
+		    T74 = T3m + T3t;
+		    T75 = T73 - T74;
+		    T7r = T73 + T74;
+	       }
+	       {
+		    E T76, T77, T6X, T70;
+		    T76 = FNMS(KP555570233, T6J, KP831469612 * T6K);
+		    T77 = FMA(KP555570233, T6G, KP831469612 * T6H);
+		    T78 = T76 - T77;
+		    T7i = T77 + T76;
+		    T6X = T6V - T6W;
+		    T70 = T6Y - T6Z;
+		    T71 = FNMS(KP881921264, T70, KP471396736 * T6X);
+		    T7d = FMA(KP881921264, T6X, KP471396736 * T70);
+	       }
+	       {
+		    E T6N, T72, T7f, T7g;
+		    T6N = T6F + T6M;
+		    T72 = T6U + T71;
+		    ro[WS(os, 43)] = T6N - T72;
+		    ro[WS(os, 11)] = T6N + T72;
+		    T7f = T75 + T78;
+		    T7g = T7c + T7d;
+		    io[WS(os, 43)] = T7f - T7g;
+		    io[WS(os, 11)] = T7f + T7g;
+	       }
+	       {
+		    E T79, T7a, T7b, T7e;
+		    T79 = T75 - T78;
+		    T7a = T71 - T6U;
+		    io[WS(os, 59)] = T79 - T7a;
+		    io[WS(os, 27)] = T79 + T7a;
+		    T7b = T6F - T6M;
+		    T7e = T7c - T7d;
+		    ro[WS(os, 59)] = T7b - T7e;
+		    ro[WS(os, 27)] = T7b + T7e;
+	       }
+	       {
+		    E T7j, T7q, T7z, T7A;
+		    T7j = T7h + T7i;
+		    T7q = T7m + T7p;
+		    ro[WS(os, 35)] = T7j - T7q;
+		    ro[WS(os, 3)] = T7j + T7q;
+		    T7z = T7r + T7s;
+		    T7A = T7w + T7x;
+		    io[WS(os, 35)] = T7z - T7A;
+		    io[WS(os, 3)] = T7z + T7A;
+	       }
+	       {
+		    E T7t, T7u, T7v, T7y;
+		    T7t = T7r - T7s;
+		    T7u = T7p - T7m;
+		    io[WS(os, 51)] = T7t - T7u;
+		    io[WS(os, 19)] = T7t + T7u;
+		    T7v = T7h - T7i;
+		    T7y = T7w - T7x;
+		    ro[WS(os, 51)] = T7v - T7y;
+		    ro[WS(os, 19)] = T7v + T7y;
+	       }
+	  }
+	  {
+	       E T9j, T9V, Ta0, Taa, Ta3, Tab, T9q, Ta6, T9y, T9Q, T9J, Ta5, T9M, T9W, T9F;
+	       E T9R;
+	       {
+		    E T9h, T9i, T9Y, T9Z;
+		    T9h = T7B + T7C;
+		    T9i = T8J + T8I;
+		    T9j = T9h - T9i;
+		    T9V = T9h + T9i;
+		    T9Y = T9s + T9t;
+		    T9Z = T9v + T9w;
+		    Ta0 = FMA(KP995184726, T9Y, KP098017140 * T9Z);
+		    Taa = FNMS(KP098017140, T9Y, KP995184726 * T9Z);
+	       }
+	       {
+		    E Ta1, Ta2, T9m, T9p;
+		    Ta1 = T9z + T9A;
+		    Ta2 = T9C + T9D;
+		    Ta3 = FNMS(KP098017140, Ta2, KP995184726 * Ta1);
+		    Tab = FMA(KP098017140, Ta1, KP995184726 * Ta2);
+		    T9m = FNMS(KP195090322, T9l, KP980785280 * T9k);
+		    T9p = FMA(KP195090322, T9n, KP980785280 * T9o);
+		    T9q = T9m - T9p;
+		    Ta6 = T9m + T9p;
+	       }
+	       {
+		    E T9u, T9x, T9H, T9I;
+		    T9u = T9s - T9t;
+		    T9x = T9v - T9w;
+		    T9y = FMA(KP634393284, T9u, KP773010453 * T9x);
+		    T9Q = FNMS(KP773010453, T9u, KP634393284 * T9x);
+		    T9H = T8F + T8G;
+		    T9I = T7G + T7J;
+		    T9J = T9H - T9I;
+		    Ta5 = T9H + T9I;
+	       }
+	       {
+		    E T9K, T9L, T9B, T9E;
+		    T9K = FNMS(KP195090322, T9o, KP980785280 * T9n);
+		    T9L = FMA(KP980785280, T9l, KP195090322 * T9k);
+		    T9M = T9K - T9L;
+		    T9W = T9L + T9K;
+		    T9B = T9z - T9A;
+		    T9E = T9C - T9D;
+		    T9F = FNMS(KP773010453, T9E, KP634393284 * T9B);
+		    T9R = FMA(KP773010453, T9B, KP634393284 * T9E);
+	       }
+	       {
+		    E T9r, T9G, T9T, T9U;
+		    T9r = T9j + T9q;
+		    T9G = T9y + T9F;
+		    ro[WS(os, 41)] = T9r - T9G;
+		    ro[WS(os, 9)] = T9r + T9G;
+		    T9T = T9J + T9M;
+		    T9U = T9Q + T9R;
+		    io[WS(os, 41)] = T9T - T9U;
+		    io[WS(os, 9)] = T9T + T9U;
+	       }
+	       {
+		    E T9N, T9O, T9P, T9S;
+		    T9N = T9J - T9M;
+		    T9O = T9F - T9y;
+		    io[WS(os, 57)] = T9N - T9O;
+		    io[WS(os, 25)] = T9N + T9O;
+		    T9P = T9j - T9q;
+		    T9S = T9Q - T9R;
+		    ro[WS(os, 57)] = T9P - T9S;
+		    ro[WS(os, 25)] = T9P + T9S;
+	       }
+	       {
+		    E T9X, Ta4, Tad, Tae;
+		    T9X = T9V + T9W;
+		    Ta4 = Ta0 + Ta3;
+		    ro[WS(os, 33)] = T9X - Ta4;
+		    ro[WS(os, 1)] = T9X + Ta4;
+		    Tad = Ta5 + Ta6;
+		    Tae = Taa + Tab;
+		    io[WS(os, 33)] = Tad - Tae;
+		    io[WS(os, 1)] = Tad + Tae;
+	       }
+	       {
+		    E Ta7, Ta8, Ta9, Tac;
+		    Ta7 = Ta5 - Ta6;
+		    Ta8 = Ta3 - Ta0;
+		    io[WS(os, 49)] = Ta7 - Ta8;
+		    io[WS(os, 17)] = Ta7 + Ta8;
+		    Ta9 = T9V - T9W;
+		    Tac = Taa - Tab;
+		    ro[WS(os, 49)] = Ta9 - Tac;
+		    ro[WS(os, 17)] = Ta9 + Tac;
+	       }
+	  }
+	  {
+	       E T3v, T6j, T6o, T6y, T6r, T6z, T48, T6u, T52, T6e, T67, T6t, T6a, T6k, T5V;
+	       E T6f;
+	       {
+		    E T3f, T3u, T6m, T6n;
+		    T3f = T37 - T3e;
+		    T3u = T3m - T3t;
+		    T3v = T3f - T3u;
+		    T6j = T3f + T3u;
+		    T6m = T4q + T4N;
+		    T6n = T4X + T50;
+		    T6o = FMA(KP634393284, T6m, KP773010453 * T6n);
+		    T6y = FNMS(KP634393284, T6n, KP773010453 * T6m);
+	       }
+	       {
+		    E T6p, T6q, T3O, T47;
+		    T6p = T5j + T5G;
+		    T6q = T5Q + T5T;
+		    T6r = FNMS(KP634393284, T6q, KP773010453 * T6p);
+		    T6z = FMA(KP773010453, T6q, KP634393284 * T6p);
+		    T3O = FNMS(KP980785280, T3N, KP195090322 * T3G);
+		    T47 = FMA(KP195090322, T3Z, KP980785280 * T46);
+		    T48 = T3O - T47;
+		    T6u = T3O + T47;
+	       }
+	       {
+		    E T4O, T51, T63, T66;
+		    T4O = T4q - T4N;
+		    T51 = T4X - T50;
+		    T52 = FMA(KP995184726, T4O, KP098017140 * T51);
+		    T6e = FNMS(KP995184726, T51, KP098017140 * T4O);
+		    T63 = T5Z - T62;
+		    T66 = T64 - T65;
+		    T67 = T63 - T66;
+		    T6t = T63 + T66;
+	       }
+	       {
+		    E T68, T69, T5H, T5U;
+		    T68 = FNMS(KP980785280, T3Z, KP195090322 * T46);
+		    T69 = FMA(KP980785280, T3G, KP195090322 * T3N);
+		    T6a = T68 - T69;
+		    T6k = T69 + T68;
+		    T5H = T5j - T5G;
+		    T5U = T5Q - T5T;
+		    T5V = FNMS(KP995184726, T5U, KP098017140 * T5H);
+		    T6f = FMA(KP098017140, T5U, KP995184726 * T5H);
+	       }
+	       {
+		    E T49, T5W, T6h, T6i;
+		    T49 = T3v + T48;
+		    T5W = T52 + T5V;
+		    ro[WS(os, 47)] = T49 - T5W;
+		    ro[WS(os, 15)] = T49 + T5W;
+		    T6h = T67 + T6a;
+		    T6i = T6e + T6f;
+		    io[WS(os, 47)] = T6h - T6i;
+		    io[WS(os, 15)] = T6h + T6i;
+	       }
+	       {
+		    E T6b, T6c, T6d, T6g;
+		    T6b = T67 - T6a;
+		    T6c = T5V - T52;
+		    io[WS(os, 63)] = T6b - T6c;
+		    io[WS(os, 31)] = T6b + T6c;
+		    T6d = T3v - T48;
+		    T6g = T6e - T6f;
+		    ro[WS(os, 63)] = T6d - T6g;
+		    ro[WS(os, 31)] = T6d + T6g;
+	       }
+	       {
+		    E T6l, T6s, T6B, T6C;
+		    T6l = T6j + T6k;
+		    T6s = T6o + T6r;
+		    ro[WS(os, 39)] = T6l - T6s;
+		    ro[WS(os, 7)] = T6l + T6s;
+		    T6B = T6t + T6u;
+		    T6C = T6y + T6z;
+		    io[WS(os, 39)] = T6B - T6C;
+		    io[WS(os, 7)] = T6B + T6C;
+	       }
+	       {
+		    E T6v, T6w, T6x, T6A;
+		    T6v = T6t - T6u;
+		    T6w = T6r - T6o;
+		    io[WS(os, 55)] = T6v - T6w;
+		    io[WS(os, 23)] = T6v + T6w;
+		    T6x = T6j - T6k;
+		    T6A = T6y - T6z;
+		    ro[WS(os, 55)] = T6x - T6A;
+		    ro[WS(os, 23)] = T6x + T6A;
+	       }
+	  }
+	  {
+	       E T7L, T8X, T92, T9c, T95, T9d, T80, T98, T8k, T8S, T8L, T97, T8O, T8Y, T8D;
+	       E T8T;
+	       {
+		    E T7D, T7K, T90, T91;
+		    T7D = T7B - T7C;
+		    T7K = T7G - T7J;
+		    T7L = T7D - T7K;
+		    T8X = T7D + T7K;
+		    T90 = T84 + T8b;
+		    T91 = T8f + T8i;
+		    T92 = FMA(KP471396736, T90, KP881921264 * T91);
+		    T9c = FNMS(KP471396736, T91, KP881921264 * T90);
+	       }
+	       {
+		    E T93, T94, T7S, T7Z;
+		    T93 = T8n + T8u;
+		    T94 = T8y + T8B;
+		    T95 = FNMS(KP471396736, T94, KP881921264 * T93);
+		    T9d = FMA(KP881921264, T94, KP471396736 * T93);
+		    T7S = FNMS(KP831469612, T7R, KP555570233 * T7O);
+		    T7Z = FMA(KP831469612, T7V, KP555570233 * T7Y);
+		    T80 = T7S - T7Z;
+		    T98 = T7S + T7Z;
+	       }
+	       {
+		    E T8c, T8j, T8H, T8K;
+		    T8c = T84 - T8b;
+		    T8j = T8f - T8i;
+		    T8k = FMA(KP956940335, T8c, KP290284677 * T8j);
+		    T8S = FNMS(KP956940335, T8j, KP290284677 * T8c);
+		    T8H = T8F - T8G;
+		    T8K = T8I - T8J;
+		    T8L = T8H - T8K;
+		    T97 = T8H + T8K;
+	       }
+	       {
+		    E T8M, T8N, T8v, T8C;
+		    T8M = FNMS(KP831469612, T7Y, KP555570233 * T7V);
+		    T8N = FMA(KP555570233, T7R, KP831469612 * T7O);
+		    T8O = T8M - T8N;
+		    T8Y = T8N + T8M;
+		    T8v = T8n - T8u;
+		    T8C = T8y - T8B;
+		    T8D = FNMS(KP956940335, T8C, KP290284677 * T8v);
+		    T8T = FMA(KP290284677, T8C, KP956940335 * T8v);
+	       }
+	       {
+		    E T81, T8E, T8V, T8W;
+		    T81 = T7L + T80;
+		    T8E = T8k + T8D;
+		    ro[WS(os, 45)] = T81 - T8E;
+		    ro[WS(os, 13)] = T81 + T8E;
+		    T8V = T8L + T8O;
+		    T8W = T8S + T8T;
+		    io[WS(os, 45)] = T8V - T8W;
+		    io[WS(os, 13)] = T8V + T8W;
+	       }
+	       {
+		    E T8P, T8Q, T8R, T8U;
+		    T8P = T8L - T8O;
+		    T8Q = T8D - T8k;
+		    io[WS(os, 61)] = T8P - T8Q;
+		    io[WS(os, 29)] = T8P + T8Q;
+		    T8R = T7L - T80;
+		    T8U = T8S - T8T;
+		    ro[WS(os, 61)] = T8R - T8U;
+		    ro[WS(os, 29)] = T8R + T8U;
+	       }
+	       {
+		    E T8Z, T96, T9f, T9g;
+		    T8Z = T8X + T8Y;
+		    T96 = T92 + T95;
+		    ro[WS(os, 37)] = T8Z - T96;
+		    ro[WS(os, 5)] = T8Z + T96;
+		    T9f = T97 + T98;
+		    T9g = T9c + T9d;
+		    io[WS(os, 37)] = T9f - T9g;
+		    io[WS(os, 5)] = T9f + T9g;
+	       }
+	       {
+		    E T99, T9a, T9b, T9e;
+		    T99 = T97 - T98;
+		    T9a = T95 - T92;
+		    io[WS(os, 53)] = T99 - T9a;
+		    io[WS(os, 21)] = T99 + T9a;
+		    T9b = T8X - T8Y;
+		    T9e = T9c - T9d;
+		    ro[WS(os, 53)] = T9b - T9e;
+		    ro[WS(os, 21)] = T9b + T9e;
+	       }
+	  }
+     }
+}
+
+static void m1_64(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; i -= 1) {
+	  m1_64_0(ri, ii, ro, io, is, os);
+	  ri += ivs;
+	  ii += ivs;
+	  ro += ovs;
+	  io += ovs;
+     }
+}
+
+static const kdft_desc desc = { 64, "m1_64", {808, 144, 104, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_m1_64) (planner *p) {
+     X(kdft_register) (p, m1_64, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_10.c b/src/fftw3/dft/codelets/standard/n1_10.c
new file mode 100644
index 0000000..95fb984
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_10.c
@@ -0,0 +1,195 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:32 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 10 -name n1_10 -include n.h */
+
+/*
+ * This function contains 84 FP additions, 24 FP multiplications,
+ * (or, 72 additions, 12 multiplications, 12 fused multiply/add),
+ * 41 stack variables, and 40 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_10.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_10.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_10.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_10(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T3, Tj, TQ, T1e, TU, TV, T1c, T1b, Tm, Tp, Tq, Ta, Th, Ti, TA;
+	  E TH, T17, T14, T1f, T1g, T1h, TL, TM, TR;
+	  {
+	       E T1, T2, TO, TP;
+	       T1 = ri[0];
+	       T2 = ri[WS(is, 5)];
+	       T3 = T1 - T2;
+	       Tj = T1 + T2;
+	       TO = ii[0];
+	       TP = ii[WS(is, 5)];
+	       TQ = TO - TP;
+	       T1e = TO + TP;
+	  }
+	  {
+	       E T6, Tk, Tg, To, T9, Tl, Td, Tn;
+	       {
+		    E T4, T5, Te, Tf;
+		    T4 = ri[WS(is, 2)];
+		    T5 = ri[WS(is, 7)];
+		    T6 = T4 - T5;
+		    Tk = T4 + T5;
+		    Te = ri[WS(is, 6)];
+		    Tf = ri[WS(is, 1)];
+		    Tg = Te - Tf;
+		    To = Te + Tf;
+	       }
+	       {
+		    E T7, T8, Tb, Tc;
+		    T7 = ri[WS(is, 8)];
+		    T8 = ri[WS(is, 3)];
+		    T9 = T7 - T8;
+		    Tl = T7 + T8;
+		    Tb = ri[WS(is, 4)];
+		    Tc = ri[WS(is, 9)];
+		    Td = Tb - Tc;
+		    Tn = Tb + Tc;
+	       }
+	       TU = T6 - T9;
+	       TV = Td - Tg;
+	       T1c = Tk - Tl;
+	       T1b = Tn - To;
+	       Tm = Tk + Tl;
+	       Tp = Tn + To;
+	       Tq = Tm + Tp;
+	       Ta = T6 + T9;
+	       Th = Td + Tg;
+	       Ti = Ta + Th;
+	  }
+	  {
+	       E Tw, T15, TG, T13, Tz, T16, TD, T12;
+	       {
+		    E Tu, Tv, TE, TF;
+		    Tu = ii[WS(is, 2)];
+		    Tv = ii[WS(is, 7)];
+		    Tw = Tu - Tv;
+		    T15 = Tu + Tv;
+		    TE = ii[WS(is, 6)];
+		    TF = ii[WS(is, 1)];
+		    TG = TE - TF;
+		    T13 = TE + TF;
+	       }
+	       {
+		    E Tx, Ty, TB, TC;
+		    Tx = ii[WS(is, 8)];
+		    Ty = ii[WS(is, 3)];
+		    Tz = Tx - Ty;
+		    T16 = Tx + Ty;
+		    TB = ii[WS(is, 4)];
+		    TC = ii[WS(is, 9)];
+		    TD = TB - TC;
+		    T12 = TB + TC;
+	       }
+	       TA = Tw - Tz;
+	       TH = TD - TG;
+	       T17 = T15 - T16;
+	       T14 = T12 - T13;
+	       T1f = T15 + T16;
+	       T1g = T12 + T13;
+	       T1h = T1f + T1g;
+	       TL = Tw + Tz;
+	       TM = TD + TG;
+	       TR = TL + TM;
+	  }
+	  ro[WS(os, 5)] = T3 + Ti;
+	  io[WS(os, 5)] = TQ + TR;
+	  ro[0] = Tj + Tq;
+	  io[0] = T1e + T1h;
+	  {
+	       E TI, TK, Tt, TJ, Tr, Ts;
+	       TI = FMA(KP951056516, TA, KP587785252 * TH);
+	       TK = FNMS(KP587785252, TA, KP951056516 * TH);
+	       Tr = KP559016994 * (Ta - Th);
+	       Ts = FNMS(KP250000000, Ti, T3);
+	       Tt = Tr + Ts;
+	       TJ = Ts - Tr;
+	       ro[WS(os, 9)] = Tt - TI;
+	       ro[WS(os, 3)] = TJ + TK;
+	       ro[WS(os, 1)] = Tt + TI;
+	       ro[WS(os, 7)] = TJ - TK;
+	  }
+	  {
+	       E TW, TY, TT, TX, TN, TS;
+	       TW = FMA(KP951056516, TU, KP587785252 * TV);
+	       TY = FNMS(KP587785252, TU, KP951056516 * TV);
+	       TN = KP559016994 * (TL - TM);
+	       TS = FNMS(KP250000000, TR, TQ);
+	       TT = TN + TS;
+	       TX = TS - TN;
+	       io[WS(os, 1)] = TT - TW;
+	       io[WS(os, 7)] = TY + TX;
+	       io[WS(os, 9)] = TW + TT;
+	       io[WS(os, 3)] = TX - TY;
+	  }
+	  {
+	       E T18, T1a, T11, T19, TZ, T10;
+	       T18 = FNMS(KP587785252, T17, KP951056516 * T14);
+	       T1a = FMA(KP951056516, T17, KP587785252 * T14);
+	       TZ = FNMS(KP250000000, Tq, Tj);
+	       T10 = KP559016994 * (Tm - Tp);
+	       T11 = TZ - T10;
+	       T19 = T10 + TZ;
+	       ro[WS(os, 2)] = T11 - T18;
+	       ro[WS(os, 6)] = T19 + T1a;
+	       ro[WS(os, 8)] = T11 + T18;
+	       ro[WS(os, 4)] = T19 - T1a;
+	  }
+	  {
+	       E T1d, T1l, T1k, T1m, T1i, T1j;
+	       T1d = FNMS(KP587785252, T1c, KP951056516 * T1b);
+	       T1l = FMA(KP951056516, T1c, KP587785252 * T1b);
+	       T1i = FNMS(KP250000000, T1h, T1e);
+	       T1j = KP559016994 * (T1f - T1g);
+	       T1k = T1i - T1j;
+	       T1m = T1j + T1i;
+	       io[WS(os, 2)] = T1d + T1k;
+	       io[WS(os, 6)] = T1m - T1l;
+	       io[WS(os, 8)] = T1k - T1d;
+	       io[WS(os, 4)] = T1l + T1m;
+	  }
+     }
+}
+
+static const kdft_desc desc = { 10, "n1_10", {72, 12, 12, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_10) (planner *p) {
+     X(kdft_register) (p, n1_10, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_11.c b/src/fftw3/dft/codelets/standard/n1_11.c
new file mode 100644
index 0000000..29e8049
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_11.c
@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:32 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 11 -name n1_11 -include n.h */
+
+/*
+ * This function contains 140 FP additions, 100 FP multiplications,
+ * (or, 60 additions, 20 multiplications, 80 fused multiply/add),
+ * 41 stack variables, and 44 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_11.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_11.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_11.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_11(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP654860733, +0.654860733945285064056925072466293553183791199);
+     DK(KP142314838, +0.142314838273285140443792668616369668791051361);
+     DK(KP959492973, +0.959492973614497389890368057066327699062454848);
+     DK(KP415415013, +0.415415013001886425529274149229623203524004910);
+     DK(KP841253532, +0.841253532831181168861811648919367717513292498);
+     DK(KP989821441, +0.989821441880932732376092037776718787376519372);
+     DK(KP909631995, +0.909631995354518371411715383079028460060241051);
+     DK(KP281732556, +0.281732556841429697711417915346616899035777899);
+     DK(KP540640817, +0.540640817455597582107635954318691695431770608);
+     DK(KP755749574, +0.755749574354258283774035843972344420179717445);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, TM, T4, TG, Tk, TR, Tw, TN, T7, TK, Ta, TH, Tn, TQ, Td;
+	  E TJ, Tq, TO, Tt, TP, Tg, TI;
+	  {
+	       E T2, T3, Ti, Tj;
+	       T1 = ri[0];
+	       TM = ii[0];
+	       T2 = ri[WS(is, 1)];
+	       T3 = ri[WS(is, 10)];
+	       T4 = T2 + T3;
+	       TG = T3 - T2;
+	       Ti = ii[WS(is, 1)];
+	       Tj = ii[WS(is, 10)];
+	       Tk = Ti - Tj;
+	       TR = Ti + Tj;
+	       {
+		    E Tu, Tv, T5, T6;
+		    Tu = ii[WS(is, 2)];
+		    Tv = ii[WS(is, 9)];
+		    Tw = Tu - Tv;
+		    TN = Tu + Tv;
+		    T5 = ri[WS(is, 2)];
+		    T6 = ri[WS(is, 9)];
+		    T7 = T5 + T6;
+		    TK = T6 - T5;
+	       }
+	  }
+	  {
+	       E T8, T9, To, Tp;
+	       T8 = ri[WS(is, 3)];
+	       T9 = ri[WS(is, 8)];
+	       Ta = T8 + T9;
+	       TH = T9 - T8;
+	       {
+		    E Tl, Tm, Tb, Tc;
+		    Tl = ii[WS(is, 3)];
+		    Tm = ii[WS(is, 8)];
+		    Tn = Tl - Tm;
+		    TQ = Tl + Tm;
+		    Tb = ri[WS(is, 4)];
+		    Tc = ri[WS(is, 7)];
+		    Td = Tb + Tc;
+		    TJ = Tc - Tb;
+	       }
+	       To = ii[WS(is, 4)];
+	       Tp = ii[WS(is, 7)];
+	       Tq = To - Tp;
+	       TO = To + Tp;
+	       {
+		    E Tr, Ts, Te, Tf;
+		    Tr = ii[WS(is, 5)];
+		    Ts = ii[WS(is, 6)];
+		    Tt = Tr - Ts;
+		    TP = Tr + Ts;
+		    Te = ri[WS(is, 5)];
+		    Tf = ri[WS(is, 6)];
+		    Tg = Te + Tf;
+		    TI = Tf - Te;
+	       }
+	  }
+	  {
+	       E Tx, Th, TZ, T10;
+	       ro[0] = T1 + T4 + T7 + Ta + Td + Tg;
+	       io[0] = TM + TR + TN + TQ + TO + TP;
+	       Tx = FMA(KP755749574, Tk, KP540640817 * Tn) + FNMS(KP909631995, Tt, KP281732556 * Tq) - (KP989821441 * Tw);
+	       Th = FMA(KP841253532, Ta, T1) + FNMS(KP959492973, Td, KP415415013 * Tg) + FNMA(KP142314838, T7, KP654860733 * T4);
+	       ro[WS(os, 7)] = Th - Tx;
+	       ro[WS(os, 4)] = Th + Tx;
+	       TZ = FMA(KP755749574, TG, KP540640817 * TH) + FNMS(KP909631995, TI, KP281732556 * TJ) - (KP989821441 * TK);
+	       T10 = FMA(KP841253532, TQ, TM) + FNMS(KP959492973, TO, KP415415013 * TP) + FNMA(KP142314838, TN, KP654860733 * TR);
+	       io[WS(os, 4)] = TZ + T10;
+	       io[WS(os, 7)] = T10 - TZ;
+	       {
+		    E TX, TY, Tz, Ty;
+		    TX = FMA(KP909631995, TG, KP755749574 * TK) + FNMA(KP540640817, TI, KP989821441 * TJ) - (KP281732556 * TH);
+		    TY = FMA(KP415415013, TR, TM) + FNMS(KP142314838, TO, KP841253532 * TP) + FNMA(KP959492973, TQ, KP654860733 * TN);
+		    io[WS(os, 2)] = TX + TY;
+		    io[WS(os, 9)] = TY - TX;
+		    Tz = FMA(KP909631995, Tk, KP755749574 * Tw) + FNMA(KP540640817, Tt, KP989821441 * Tq) - (KP281732556 * Tn);
+		    Ty = FMA(KP415415013, T4, T1) + FNMS(KP142314838, Td, KP841253532 * Tg) + FNMA(KP959492973, Ta, KP654860733 * T7);
+		    ro[WS(os, 9)] = Ty - Tz;
+		    ro[WS(os, 2)] = Ty + Tz;
+	       }
+	  }
+	  {
+	       E TB, TA, TT, TU;
+	       TB = FMA(KP540640817, Tk, KP909631995 * Tw) + FMA(KP989821441, Tn, KP755749574 * Tq) + (KP281732556 * Tt);
+	       TA = FMA(KP841253532, T4, T1) + FNMS(KP959492973, Tg, KP415415013 * T7) + FNMA(KP654860733, Td, KP142314838 * Ta);
+	       ro[WS(os, 10)] = TA - TB;
+	       ro[WS(os, 1)] = TA + TB;
+	       {
+		    E TV, TW, TD, TC;
+		    TV = FMA(KP540640817, TG, KP909631995 * TK) + FMA(KP989821441, TH, KP755749574 * TJ) + (KP281732556 * TI);
+		    TW = FMA(KP841253532, TR, TM) + FNMS(KP959492973, TP, KP415415013 * TN) + FNMA(KP654860733, TO, KP142314838 * TQ);
+		    io[WS(os, 1)] = TV + TW;
+		    io[WS(os, 10)] = TW - TV;
+		    TD = FMA(KP989821441, Tk, KP540640817 * Tq) + FNMS(KP909631995, Tn, KP755749574 * Tt) - (KP281732556 * Tw);
+		    TC = FMA(KP415415013, Ta, T1) + FNMS(KP654860733, Tg, KP841253532 * Td) + FNMA(KP959492973, T7, KP142314838 * T4);
+		    ro[WS(os, 8)] = TC - TD;
+		    ro[WS(os, 3)] = TC + TD;
+	       }
+	       TT = FMA(KP989821441, TG, KP540640817 * TJ) + FNMS(KP909631995, TH, KP755749574 * TI) - (KP281732556 * TK);
+	       TU = FMA(KP415415013, TQ, TM) + FNMS(KP654860733, TP, KP841253532 * TO) + FNMA(KP959492973, TN, KP142314838 * TR);
+	       io[WS(os, 3)] = TT + TU;
+	       io[WS(os, 8)] = TU - TT;
+	       {
+		    E TL, TS, TF, TE;
+		    TL = FMA(KP281732556, TG, KP755749574 * TH) + FNMS(KP909631995, TJ, KP989821441 * TI) - (KP540640817 * TK);
+		    TS = FMA(KP841253532, TN, TM) + FNMS(KP142314838, TP, KP415415013 * TO) + FNMA(KP654860733, TQ, KP959492973 * TR);
+		    io[WS(os, 5)] = TL + TS;
+		    io[WS(os, 6)] = TS - TL;
+		    TF = FMA(KP281732556, Tk, KP755749574 * Tn) + FNMS(KP909631995, Tq, KP989821441 * Tt) - (KP540640817 * Tw);
+		    TE = FMA(KP841253532, T7, T1) + FNMS(KP142314838, Tg, KP415415013 * Td) + FNMA(KP654860733, Ta, KP959492973 * T4);
+		    ro[WS(os, 6)] = TE - TF;
+		    ro[WS(os, 5)] = TE + TF;
+	       }
+	  }
+     }
+}
+
+static const kdft_desc desc = { 11, "n1_11", {60, 20, 80, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_11) (planner *p) {
+     X(kdft_register) (p, n1_11, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_12.c b/src/fftw3/dft/codelets/standard/n1_12.c
new file mode 100644
index 0000000..cced824
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_12.c
@@ -0,0 +1,224 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:32 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 12 -name n1_12 -include n.h */
+
+/*
+ * This function contains 96 FP additions, 16 FP multiplications,
+ * (or, 88 additions, 8 multiplications, 8 fused multiply/add),
+ * 43 stack variables, and 48 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_12.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_12.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_12.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_12(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T5, TR, TA, Ts, TS, Tz, Ta, TU, TD, Tx, TV, TC, Tg, T1a, TG;
+	  E TJ, T1u, T1d, Tl, T1f, TL, TO, T1v, T1i;
+	  {
+	       E T1, T2, T3, T4;
+	       T1 = ri[0];
+	       T2 = ri[WS(is, 4)];
+	       T3 = ri[WS(is, 8)];
+	       T4 = T2 + T3;
+	       T5 = T1 + T4;
+	       TR = FNMS(KP500000000, T4, T1);
+	       TA = KP866025403 * (T3 - T2);
+	  }
+	  {
+	       E To, Tp, Tq, Tr;
+	       To = ii[0];
+	       Tp = ii[WS(is, 4)];
+	       Tq = ii[WS(is, 8)];
+	       Tr = Tp + Tq;
+	       Ts = To + Tr;
+	       TS = KP866025403 * (Tp - Tq);
+	       Tz = FNMS(KP500000000, Tr, To);
+	  }
+	  {
+	       E T6, T7, T8, T9;
+	       T6 = ri[WS(is, 6)];
+	       T7 = ri[WS(is, 10)];
+	       T8 = ri[WS(is, 2)];
+	       T9 = T7 + T8;
+	       Ta = T6 + T9;
+	       TU = FNMS(KP500000000, T9, T6);
+	       TD = KP866025403 * (T8 - T7);
+	  }
+	  {
+	       E Tt, Tu, Tv, Tw;
+	       Tt = ii[WS(is, 6)];
+	       Tu = ii[WS(is, 10)];
+	       Tv = ii[WS(is, 2)];
+	       Tw = Tu + Tv;
+	       Tx = Tt + Tw;
+	       TV = KP866025403 * (Tu - Tv);
+	       TC = FNMS(KP500000000, Tw, Tt);
+	  }
+	  {
+	       E Tc, Td, Te, Tf;
+	       Tc = ri[WS(is, 3)];
+	       Td = ri[WS(is, 7)];
+	       Te = ri[WS(is, 11)];
+	       Tf = Td + Te;
+	       Tg = Tc + Tf;
+	       T1a = KP866025403 * (Te - Td);
+	       TG = FNMS(KP500000000, Tf, Tc);
+	  }
+	  {
+	       E T1b, TH, TI, T1c;
+	       T1b = ii[WS(is, 3)];
+	       TH = ii[WS(is, 7)];
+	       TI = ii[WS(is, 11)];
+	       T1c = TH + TI;
+	       TJ = KP866025403 * (TH - TI);
+	       T1u = T1b + T1c;
+	       T1d = FNMS(KP500000000, T1c, T1b);
+	  }
+	  {
+	       E Th, Ti, Tj, Tk;
+	       Th = ri[WS(is, 9)];
+	       Ti = ri[WS(is, 1)];
+	       Tj = ri[WS(is, 5)];
+	       Tk = Ti + Tj;
+	       Tl = Th + Tk;
+	       T1f = KP866025403 * (Tj - Ti);
+	       TL = FNMS(KP500000000, Tk, Th);
+	  }
+	  {
+	       E T1g, TM, TN, T1h;
+	       T1g = ii[WS(is, 9)];
+	       TM = ii[WS(is, 1)];
+	       TN = ii[WS(is, 5)];
+	       T1h = TM + TN;
+	       TO = KP866025403 * (TM - TN);
+	       T1v = T1g + T1h;
+	       T1i = FNMS(KP500000000, T1h, T1g);
+	  }
+	  {
+	       E Tb, Tm, T1t, T1w;
+	       Tb = T5 + Ta;
+	       Tm = Tg + Tl;
+	       ro[WS(os, 6)] = Tb - Tm;
+	       ro[0] = Tb + Tm;
+	       {
+		    E T1x, T1y, Tn, Ty;
+		    T1x = Ts + Tx;
+		    T1y = T1u + T1v;
+		    io[WS(os, 6)] = T1x - T1y;
+		    io[0] = T1x + T1y;
+		    Tn = Tg - Tl;
+		    Ty = Ts - Tx;
+		    io[WS(os, 3)] = Tn + Ty;
+		    io[WS(os, 9)] = Ty - Tn;
+	       }
+	       T1t = T5 - Ta;
+	       T1w = T1u - T1v;
+	       ro[WS(os, 3)] = T1t - T1w;
+	       ro[WS(os, 9)] = T1t + T1w;
+	       {
+		    E T11, T1l, T1k, T1m, T14, T18, T17, T19;
+		    {
+			 E TZ, T10, T1e, T1j;
+			 TZ = TA + Tz;
+			 T10 = TD + TC;
+			 T11 = TZ - T10;
+			 T1l = TZ + T10;
+			 T1e = T1a + T1d;
+			 T1j = T1f + T1i;
+			 T1k = T1e - T1j;
+			 T1m = T1e + T1j;
+		    }
+		    {
+			 E T12, T13, T15, T16;
+			 T12 = TG + TJ;
+			 T13 = TL + TO;
+			 T14 = T12 - T13;
+			 T18 = T12 + T13;
+			 T15 = TR + TS;
+			 T16 = TU + TV;
+			 T17 = T15 + T16;
+			 T19 = T15 - T16;
+		    }
+		    io[WS(os, 1)] = T11 - T14;
+		    ro[WS(os, 1)] = T19 + T1k;
+		    io[WS(os, 7)] = T11 + T14;
+		    ro[WS(os, 7)] = T19 - T1k;
+		    ro[WS(os, 10)] = T17 - T18;
+		    io[WS(os, 10)] = T1l - T1m;
+		    ro[WS(os, 4)] = T17 + T18;
+		    io[WS(os, 4)] = T1l + T1m;
+	       }
+	       {
+		    E TF, T1r, T1q, T1s, TQ, TY, TX, T1n;
+		    {
+			 E TB, TE, T1o, T1p;
+			 TB = Tz - TA;
+			 TE = TC - TD;
+			 TF = TB - TE;
+			 T1r = TB + TE;
+			 T1o = T1d - T1a;
+			 T1p = T1i - T1f;
+			 T1q = T1o - T1p;
+			 T1s = T1o + T1p;
+		    }
+		    {
+			 E TK, TP, TT, TW;
+			 TK = TG - TJ;
+			 TP = TL - TO;
+			 TQ = TK - TP;
+			 TY = TK + TP;
+			 TT = TR - TS;
+			 TW = TU - TV;
+			 TX = TT + TW;
+			 T1n = TT - TW;
+		    }
+		    io[WS(os, 5)] = TF - TQ;
+		    ro[WS(os, 5)] = T1n + T1q;
+		    io[WS(os, 11)] = TF + TQ;
+		    ro[WS(os, 11)] = T1n - T1q;
+		    ro[WS(os, 2)] = TX - TY;
+		    io[WS(os, 2)] = T1r - T1s;
+		    ro[WS(os, 8)] = TX + TY;
+		    io[WS(os, 8)] = T1r + T1s;
+	       }
+	  }
+     }
+}
+
+static const kdft_desc desc = { 12, "n1_12", {88, 8, 8, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_12) (planner *p) {
+     X(kdft_register) (p, n1_12, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_13.c b/src/fftw3/dft/codelets/standard/n1_13.c
new file mode 100644
index 0000000..3f8c804
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_13.c
@@ -0,0 +1,352 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:33 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 13 -name n1_13 -include n.h */
+
+/*
+ * This function contains 176 FP additions, 68 FP multiplications,
+ * (or, 138 additions, 30 multiplications, 38 fused multiply/add),
+ * 71 stack variables, and 52 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_13.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_13.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_13.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_13(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP083333333, +0.083333333333333333333333333333333333333333333);
+     DK(KP251768516, +0.251768516431883313623436926934233488546674281);
+     DK(KP075902986, +0.075902986037193865983102897245103540356428373);
+     DK(KP132983124, +0.132983124607418643793760531921092974399165133);
+     DK(KP258260390, +0.258260390311744861420450644284508567852516811);
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     DK(KP300238635, +0.300238635966332641462884626667381504676006424);
+     DK(KP011599105, +0.011599105605768290721655456654083252189827041);
+     DK(KP156891391, +0.156891391051584611046832726756003269660212636);
+     DK(KP256247671, +0.256247671582936600958684654061725059144125175);
+     DK(KP174138601, +0.174138601152135905005660794929264742616964676);
+     DK(KP575140729, +0.575140729474003121368385547455453388461001608);
+     DK(KP503537032, +0.503537032863766627246873853868466977093348562);
+     DK(KP113854479, +0.113854479055790798974654345867655310534642560);
+     DK(KP265966249, +0.265966249214837287587521063842185948798330267);
+     DK(KP387390585, +0.387390585467617292130675966426762851778775217);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP300462606, +0.300462606288665774426601772289207995520941381);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T1q, Tt, Tu, To, T22, T20, T24, TF, TH, TA, TI, T1X, T25, T2a;
+	  E T2d, T18, T1n, T2k, T2n, T1l, T1r, T1f, T1o, T2h, T2m;
+	  T1 = ri[0];
+	  T1q = ii[0];
+	  {
+	       E Tf, Tp, Tb, TC, Tx, T6, TB, Tw, Ti, Tq, Tl, Tr, Tm, Ts, Td;
+	       E Te, Tc, Tn;
+	       Td = ri[WS(is, 8)];
+	       Te = ri[WS(is, 5)];
+	       Tf = Td + Te;
+	       Tp = Td - Te;
+	       {
+		    E T7, T8, T9, Ta;
+		    T7 = ri[WS(is, 12)];
+		    T8 = ri[WS(is, 10)];
+		    T9 = ri[WS(is, 4)];
+		    Ta = T8 + T9;
+		    Tb = T7 + Ta;
+		    TC = T8 - T9;
+		    Tx = FNMS(KP500000000, Ta, T7);
+	       }
+	       {
+		    E T2, T3, T4, T5;
+		    T2 = ri[WS(is, 1)];
+		    T3 = ri[WS(is, 3)];
+		    T4 = ri[WS(is, 9)];
+		    T5 = T3 + T4;
+		    T6 = T2 + T5;
+		    TB = T3 - T4;
+		    Tw = FNMS(KP500000000, T5, T2);
+	       }
+	       {
+		    E Tg, Th, Tj, Tk;
+		    Tg = ri[WS(is, 11)];
+		    Th = ri[WS(is, 6)];
+		    Ti = Tg + Th;
+		    Tq = Tg - Th;
+		    Tj = ri[WS(is, 7)];
+		    Tk = ri[WS(is, 2)];
+		    Tl = Tj + Tk;
+		    Tr = Tj - Tk;
+	       }
+	       Tm = Ti + Tl;
+	       Ts = Tq + Tr;
+	       Tt = Tp + Ts;
+	       Tu = T6 - Tb;
+	       Tc = T6 + Tb;
+	       Tn = Tf + Tm;
+	       To = Tc + Tn;
+	       T22 = KP300462606 * (Tc - Tn);
+	       {
+		    E T1Y, T1Z, TD, TE;
+		    T1Y = TB + TC;
+		    T1Z = Tq - Tr;
+		    T20 = T1Y - T1Z;
+		    T24 = T1Y + T1Z;
+		    TD = KP866025403 * (TB - TC);
+		    TE = FNMS(KP500000000, Ts, Tp);
+		    TF = TD - TE;
+		    TH = TD + TE;
+	       }
+	       {
+		    E Ty, Tz, T1V, T1W;
+		    Ty = Tw - Tx;
+		    Tz = KP866025403 * (Ti - Tl);
+		    TA = Ty + Tz;
+		    TI = Ty - Tz;
+		    T1V = Tw + Tx;
+		    T1W = FNMS(KP500000000, Tm, Tf);
+		    T1X = T1V - T1W;
+		    T25 = T1V + T1W;
+	       }
+	  }
+	  {
+	       E TZ, T2b, TV, T1i, T1a, TQ, T1h, T19, T12, T1d, T15, T1c, T16, T2c, TX;
+	       E TY, TW, T17;
+	       TX = ii[WS(is, 8)];
+	       TY = ii[WS(is, 5)];
+	       TZ = TX + TY;
+	       T2b = TX - TY;
+	       {
+		    E TR, TS, TT, TU;
+		    TR = ii[WS(is, 12)];
+		    TS = ii[WS(is, 10)];
+		    TT = ii[WS(is, 4)];
+		    TU = TS + TT;
+		    TV = FNMS(KP500000000, TU, TR);
+		    T1i = TR + TU;
+		    T1a = TS - TT;
+	       }
+	       {
+		    E TM, TN, TO, TP;
+		    TM = ii[WS(is, 1)];
+		    TN = ii[WS(is, 3)];
+		    TO = ii[WS(is, 9)];
+		    TP = TN + TO;
+		    TQ = FNMS(KP500000000, TP, TM);
+		    T1h = TM + TP;
+		    T19 = TN - TO;
+	       }
+	       {
+		    E T10, T11, T13, T14;
+		    T10 = ii[WS(is, 11)];
+		    T11 = ii[WS(is, 6)];
+		    T12 = T10 + T11;
+		    T1d = T10 - T11;
+		    T13 = ii[WS(is, 7)];
+		    T14 = ii[WS(is, 2)];
+		    T15 = T13 + T14;
+		    T1c = T13 - T14;
+	       }
+	       T16 = T12 + T15;
+	       T2c = T1d + T1c;
+	       T2a = T1h - T1i;
+	       T2d = T2b + T2c;
+	       TW = TQ + TV;
+	       T17 = FNMS(KP500000000, T16, TZ);
+	       T18 = TW - T17;
+	       T1n = TW + T17;
+	       {
+		    E T2i, T2j, T1j, T1k;
+		    T2i = TQ - TV;
+		    T2j = KP866025403 * (T15 - T12);
+		    T2k = T2i + T2j;
+		    T2n = T2i - T2j;
+		    T1j = T1h + T1i;
+		    T1k = TZ + T16;
+		    T1l = KP300462606 * (T1j - T1k);
+		    T1r = T1j + T1k;
+	       }
+	       {
+		    E T1b, T1e, T2f, T2g;
+		    T1b = T19 + T1a;
+		    T1e = T1c - T1d;
+		    T1f = T1b + T1e;
+		    T1o = T1e - T1b;
+		    T2f = FNMS(KP500000000, T2c, T2b);
+		    T2g = KP866025403 * (T1a - T19);
+		    T2h = T2f - T2g;
+		    T2m = T2g + T2f;
+	       }
+	  }
+	  ro[0] = T1 + To;
+	  io[0] = T1q + T1r;
+	  {
+	       E T1D, T1N, T1y, T1x, T1E, T1O, Tv, TK, T1J, T1Q, T1m, T1R, T1t, T1I, TG;
+	       E TJ;
+	       {
+		    E T1B, T1C, T1v, T1w;
+		    T1B = FMA(KP387390585, T1f, KP265966249 * T18);
+		    T1C = FMA(KP113854479, T1o, KP503537032 * T1n);
+		    T1D = T1B + T1C;
+		    T1N = T1C - T1B;
+		    T1y = FMA(KP575140729, Tu, KP174138601 * Tt);
+		    T1v = FNMS(KP156891391, TH, KP256247671 * TI);
+		    T1w = FMA(KP011599105, TF, KP300238635 * TA);
+		    T1x = T1v - T1w;
+		    T1E = T1y + T1x;
+		    T1O = KP1_732050807 * (T1v + T1w);
+	       }
+	       Tv = FNMS(KP174138601, Tu, KP575140729 * Tt);
+	       TG = FNMS(KP300238635, TF, KP011599105 * TA);
+	       TJ = FMA(KP256247671, TH, KP156891391 * TI);
+	       TK = TG - TJ;
+	       T1J = KP1_732050807 * (TJ + TG);
+	       T1Q = Tv - TK;
+	       {
+		    E T1g, T1H, T1p, T1s, T1G;
+		    T1g = FNMS(KP132983124, T1f, KP258260390 * T18);
+		    T1H = T1l - T1g;
+		    T1p = FNMS(KP251768516, T1o, KP075902986 * T1n);
+		    T1s = FNMS(KP083333333, T1r, T1q);
+		    T1G = T1s - T1p;
+		    T1m = FMA(KP2_000000000, T1g, T1l);
+		    T1R = T1H + T1G;
+		    T1t = FMA(KP2_000000000, T1p, T1s);
+		    T1I = T1G - T1H;
+	       }
+	       {
+		    E TL, T1u, T1P, T1S;
+		    TL = FMA(KP2_000000000, TK, Tv);
+		    T1u = T1m + T1t;
+		    io[WS(os, 1)] = TL + T1u;
+		    io[WS(os, 12)] = T1u - TL;
+		    {
+			 E T1z, T1A, T1T, T1U;
+			 T1z = FMS(KP2_000000000, T1x, T1y);
+			 T1A = T1t - T1m;
+			 io[WS(os, 5)] = T1z + T1A;
+			 io[WS(os, 8)] = T1A - T1z;
+			 T1T = T1R - T1Q;
+			 T1U = T1O + T1N;
+			 io[WS(os, 4)] = T1T - T1U;
+			 io[WS(os, 10)] = T1U + T1T;
+		    }
+		    T1P = T1N - T1O;
+		    T1S = T1Q + T1R;
+		    io[WS(os, 3)] = T1P + T1S;
+		    io[WS(os, 9)] = T1S - T1P;
+		    {
+			 E T1L, T1M, T1F, T1K;
+			 T1L = T1J + T1I;
+			 T1M = T1E + T1D;
+			 io[WS(os, 6)] = T1L - T1M;
+			 io[WS(os, 11)] = T1M + T1L;
+			 T1F = T1D - T1E;
+			 T1K = T1I - T1J;
+			 io[WS(os, 2)] = T1F + T1K;
+			 io[WS(os, 7)] = T1K - T1F;
+		    }
+	       }
+	  }
+	  {
+	       E T2y, T2I, T2J, T2K, T2B, T2L, T2e, T2p, T2u, T2G, T23, T2F, T28, T2t, T2l;
+	       E T2o;
+	       {
+		    E T2w, T2x, T2z, T2A;
+		    T2w = FMA(KP387390585, T20, KP265966249 * T1X);
+		    T2x = FNMS(KP503537032, T25, KP113854479 * T24);
+		    T2y = T2w + T2x;
+		    T2I = T2w - T2x;
+		    T2J = FMA(KP575140729, T2a, KP174138601 * T2d);
+		    T2z = FNMS(KP300238635, T2n, KP011599105 * T2m);
+		    T2A = FNMS(KP156891391, T2h, KP256247671 * T2k);
+		    T2K = T2z + T2A;
+		    T2B = KP1_732050807 * (T2z - T2A);
+		    T2L = T2J + T2K;
+	       }
+	       T2e = FNMS(KP575140729, T2d, KP174138601 * T2a);
+	       T2l = FMA(KP256247671, T2h, KP156891391 * T2k);
+	       T2o = FMA(KP300238635, T2m, KP011599105 * T2n);
+	       T2p = T2l - T2o;
+	       T2u = T2e - T2p;
+	       T2G = KP1_732050807 * (T2o + T2l);
+	       {
+		    E T21, T2r, T26, T27, T2s;
+		    T21 = FNMS(KP132983124, T20, KP258260390 * T1X);
+		    T2r = T22 - T21;
+		    T26 = FMA(KP251768516, T24, KP075902986 * T25);
+		    T27 = FNMS(KP083333333, To, T1);
+		    T2s = T27 - T26;
+		    T23 = FMA(KP2_000000000, T21, T22);
+		    T2F = T2s - T2r;
+		    T28 = FMA(KP2_000000000, T26, T27);
+		    T2t = T2r + T2s;
+	       }
+	       {
+		    E T29, T2q, T2N, T2O;
+		    T29 = T23 + T28;
+		    T2q = FMA(KP2_000000000, T2p, T2e);
+		    ro[WS(os, 12)] = T29 - T2q;
+		    ro[WS(os, 1)] = T29 + T2q;
+		    {
+			 E T2v, T2C, T2P, T2Q;
+			 T2v = T2t - T2u;
+			 T2C = T2y - T2B;
+			 ro[WS(os, 10)] = T2v - T2C;
+			 ro[WS(os, 4)] = T2v + T2C;
+			 T2P = T28 - T23;
+			 T2Q = FMS(KP2_000000000, T2K, T2J);
+			 ro[WS(os, 5)] = T2P - T2Q;
+			 ro[WS(os, 8)] = T2P + T2Q;
+		    }
+		    T2N = T2F - T2G;
+		    T2O = T2L - T2I;
+		    ro[WS(os, 11)] = T2N - T2O;
+		    ro[WS(os, 6)] = T2N + T2O;
+		    {
+			 E T2H, T2M, T2D, T2E;
+			 T2H = T2F + T2G;
+			 T2M = T2I + T2L;
+			 ro[WS(os, 7)] = T2H - T2M;
+			 ro[WS(os, 2)] = T2H + T2M;
+			 T2D = T2t + T2u;
+			 T2E = T2y + T2B;
+			 ro[WS(os, 3)] = T2D - T2E;
+			 ro[WS(os, 9)] = T2D + T2E;
+		    }
+	       }
+	  }
+     }
+}
+
+static const kdft_desc desc = { 13, "n1_13", {138, 30, 38, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_13) (planner *p) {
+     X(kdft_register) (p, n1_13, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_14.c b/src/fftw3/dft/codelets/standard/n1_14.c
new file mode 100644
index 0000000..a5d5928
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_14.c
@@ -0,0 +1,249 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:34 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 14 -name n1_14 -include n.h */
+
+/*
+ * This function contains 148 FP additions, 72 FP multiplications,
+ * (or, 100 additions, 24 multiplications, 48 fused multiply/add),
+ * 43 stack variables, and 56 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_14.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_14.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_14.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_14(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP222520933, +0.222520933956314404288902564496794759466355569);
+     DK(KP900968867, +0.900968867902419126236102319507445051165919162);
+     DK(KP623489801, +0.623489801858733530525004884004239810632274731);
+     DK(KP433883739, +0.433883739117558120475768332848358754609990728);
+     DK(KP781831482, +0.781831482468029808708444526674057750232334519);
+     DK(KP974927912, +0.974927912181823607018131682993931217232785801);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T3, Tp, T16, T1f, Ta, T1q, Ts, T10, TG, T1z, T19, T1i, Th, T1s, Tv;
+	  E T12, TU, T1B, T17, T1o, To, T1r, Ty, T11, TN, T1A, T18, T1l;
+	  {
+	       E T1, T2, T14, T15;
+	       T1 = ri[0];
+	       T2 = ri[WS(is, 7)];
+	       T3 = T1 - T2;
+	       Tp = T1 + T2;
+	       T14 = ii[0];
+	       T15 = ii[WS(is, 7)];
+	       T16 = T14 - T15;
+	       T1f = T14 + T15;
+	  }
+	  {
+	       E T6, Tq, T9, Tr;
+	       {
+		    E T4, T5, T7, T8;
+		    T4 = ri[WS(is, 2)];
+		    T5 = ri[WS(is, 9)];
+		    T6 = T4 - T5;
+		    Tq = T4 + T5;
+		    T7 = ri[WS(is, 12)];
+		    T8 = ri[WS(is, 5)];
+		    T9 = T7 - T8;
+		    Tr = T7 + T8;
+	       }
+	       Ta = T6 + T9;
+	       T1q = Tr - Tq;
+	       Ts = Tq + Tr;
+	       T10 = T9 - T6;
+	  }
+	  {
+	       E TC, T1g, TF, T1h;
+	       {
+		    E TA, TB, TD, TE;
+		    TA = ii[WS(is, 2)];
+		    TB = ii[WS(is, 9)];
+		    TC = TA - TB;
+		    T1g = TA + TB;
+		    TD = ii[WS(is, 12)];
+		    TE = ii[WS(is, 5)];
+		    TF = TD - TE;
+		    T1h = TD + TE;
+	       }
+	       TG = TC - TF;
+	       T1z = T1g - T1h;
+	       T19 = TC + TF;
+	       T1i = T1g + T1h;
+	  }
+	  {
+	       E Td, Tt, Tg, Tu;
+	       {
+		    E Tb, Tc, Te, Tf;
+		    Tb = ri[WS(is, 4)];
+		    Tc = ri[WS(is, 11)];
+		    Td = Tb - Tc;
+		    Tt = Tb + Tc;
+		    Te = ri[WS(is, 10)];
+		    Tf = ri[WS(is, 3)];
+		    Tg = Te - Tf;
+		    Tu = Te + Tf;
+	       }
+	       Th = Td + Tg;
+	       T1s = Tt - Tu;
+	       Tv = Tt + Tu;
+	       T12 = Tg - Td;
+	  }
+	  {
+	       E TQ, T1m, TT, T1n;
+	       {
+		    E TO, TP, TR, TS;
+		    TO = ii[WS(is, 4)];
+		    TP = ii[WS(is, 11)];
+		    TQ = TO - TP;
+		    T1m = TO + TP;
+		    TR = ii[WS(is, 10)];
+		    TS = ii[WS(is, 3)];
+		    TT = TR - TS;
+		    T1n = TR + TS;
+	       }
+	       TU = TQ - TT;
+	       T1B = T1n - T1m;
+	       T17 = TQ + TT;
+	       T1o = T1m + T1n;
+	  }
+	  {
+	       E Tk, Tw, Tn, Tx;
+	       {
+		    E Ti, Tj, Tl, Tm;
+		    Ti = ri[WS(is, 6)];
+		    Tj = ri[WS(is, 13)];
+		    Tk = Ti - Tj;
+		    Tw = Ti + Tj;
+		    Tl = ri[WS(is, 8)];
+		    Tm = ri[WS(is, 1)];
+		    Tn = Tl - Tm;
+		    Tx = Tl + Tm;
+	       }
+	       To = Tk + Tn;
+	       T1r = Tw - Tx;
+	       Ty = Tw + Tx;
+	       T11 = Tn - Tk;
+	  }
+	  {
+	       E TJ, T1j, TM, T1k;
+	       {
+		    E TH, TI, TK, TL;
+		    TH = ii[WS(is, 6)];
+		    TI = ii[WS(is, 13)];
+		    TJ = TH - TI;
+		    T1j = TH + TI;
+		    TK = ii[WS(is, 8)];
+		    TL = ii[WS(is, 1)];
+		    TM = TK - TL;
+		    T1k = TK + TL;
+	       }
+	       TN = TJ - TM;
+	       T1A = T1k - T1j;
+	       T18 = TJ + TM;
+	       T1l = T1j + T1k;
+	  }
+	  ro[WS(os, 7)] = T3 + Ta + Th + To;
+	  io[WS(os, 7)] = T16 + T19 + T17 + T18;
+	  ro[0] = Tp + Ts + Tv + Ty;
+	  io[0] = T1f + T1i + T1o + T1l;
+	  {
+	       E TV, Tz, T1e, T1d;
+	       TV = FNMS(KP781831482, TN, KP974927912 * TG) - (KP433883739 * TU);
+	       Tz = FMA(KP623489801, To, T3) + FNMA(KP900968867, Th, KP222520933 * Ta);
+	       ro[WS(os, 5)] = Tz - TV;
+	       ro[WS(os, 9)] = Tz + TV;
+	       T1e = FNMS(KP781831482, T11, KP974927912 * T10) - (KP433883739 * T12);
+	       T1d = FMA(KP623489801, T18, T16) + FNMA(KP900968867, T17, KP222520933 * T19);
+	       io[WS(os, 5)] = T1d - T1e;
+	       io[WS(os, 9)] = T1e + T1d;
+	  }
+	  {
+	       E TX, TW, T1b, T1c;
+	       TX = FMA(KP781831482, TG, KP974927912 * TU) + (KP433883739 * TN);
+	       TW = FMA(KP623489801, Ta, T3) + FNMA(KP900968867, To, KP222520933 * Th);
+	       ro[WS(os, 13)] = TW - TX;
+	       ro[WS(os, 1)] = TW + TX;
+	       T1b = FMA(KP781831482, T10, KP974927912 * T12) + (KP433883739 * T11);
+	       T1c = FMA(KP623489801, T19, T16) + FNMA(KP900968867, T18, KP222520933 * T17);
+	       io[WS(os, 1)] = T1b + T1c;
+	       io[WS(os, 13)] = T1c - T1b;
+	  }
+	  {
+	       E TZ, TY, T13, T1a;
+	       TZ = FMA(KP433883739, TG, KP974927912 * TN) - (KP781831482 * TU);
+	       TY = FMA(KP623489801, Th, T3) + FNMA(KP222520933, To, KP900968867 * Ta);
+	       ro[WS(os, 11)] = TY - TZ;
+	       ro[WS(os, 3)] = TY + TZ;
+	       T13 = FMA(KP433883739, T10, KP974927912 * T11) - (KP781831482 * T12);
+	       T1a = FMA(KP623489801, T17, T16) + FNMA(KP222520933, T18, KP900968867 * T19);
+	       io[WS(os, 3)] = T13 + T1a;
+	       io[WS(os, 11)] = T1a - T13;
+	  }
+	  {
+	       E T1t, T1p, T1C, T1y;
+	       T1t = FNMS(KP433883739, T1r, KP781831482 * T1q) - (KP974927912 * T1s);
+	       T1p = FMA(KP623489801, T1i, T1f) + FNMA(KP900968867, T1l, KP222520933 * T1o);
+	       io[WS(os, 6)] = T1p - T1t;
+	       io[WS(os, 8)] = T1t + T1p;
+	       T1C = FNMS(KP433883739, T1A, KP781831482 * T1z) - (KP974927912 * T1B);
+	       T1y = FMA(KP623489801, Ts, Tp) + FNMA(KP900968867, Ty, KP222520933 * Tv);
+	       ro[WS(os, 6)] = T1y - T1C;
+	       ro[WS(os, 8)] = T1y + T1C;
+	  }
+	  {
+	       E T1v, T1u, T1E, T1D;
+	       T1v = FMA(KP433883739, T1q, KP781831482 * T1s) - (KP974927912 * T1r);
+	       T1u = FMA(KP623489801, T1o, T1f) + FNMA(KP222520933, T1l, KP900968867 * T1i);
+	       io[WS(os, 4)] = T1u - T1v;
+	       io[WS(os, 10)] = T1v + T1u;
+	       T1E = FMA(KP433883739, T1z, KP781831482 * T1B) - (KP974927912 * T1A);
+	       T1D = FMA(KP623489801, Tv, Tp) + FNMA(KP222520933, Ty, KP900968867 * Ts);
+	       ro[WS(os, 4)] = T1D - T1E;
+	       ro[WS(os, 10)] = T1D + T1E;
+	  }
+	  {
+	       E T1w, T1x, T1G, T1F;
+	       T1w = FMA(KP974927912, T1q, KP433883739 * T1s) + (KP781831482 * T1r);
+	       T1x = FMA(KP623489801, T1l, T1f) + FNMA(KP900968867, T1o, KP222520933 * T1i);
+	       io[WS(os, 2)] = T1w + T1x;
+	       io[WS(os, 12)] = T1x - T1w;
+	       T1G = FMA(KP974927912, T1z, KP433883739 * T1B) + (KP781831482 * T1A);
+	       T1F = FMA(KP623489801, Ty, Tp) + FNMA(KP900968867, Tv, KP222520933 * Ts);
+	       ro[WS(os, 12)] = T1F - T1G;
+	       ro[WS(os, 2)] = T1F + T1G;
+	  }
+     }
+}
+
+static const kdft_desc desc = { 14, "n1_14", {100, 24, 48, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_14) (planner *p) {
+     X(kdft_register) (p, n1_14, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_15.c b/src/fftw3/dft/codelets/standard/n1_15.c
new file mode 100644
index 0000000..5fbb0f4
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_15.c
@@ -0,0 +1,291 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:34 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 15 -name n1_15 -include n.h */
+
+/*
+ * This function contains 156 FP additions, 56 FP multiplications,
+ * (or, 128 additions, 28 multiplications, 28 fused multiply/add),
+ * 69 stack variables, and 60 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_15.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_15.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_15.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_15(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T5, T2l, Tx, TV, T1C, T20, Tl, Tq, Tr, TN, TS, TT, T2c, T2d, T2n;
+	  E T1O, T1P, T22, T1l, T1q, T1w, TZ, T10, T11, Ta, Tf, Tg, TC, TH, TI;
+	  E T2f, T2g, T2m, T1R, T1S, T21, T1a, T1f, T1v, TW, TX, TY;
+	  {
+	       E T1, T1z, T4, T1y, Tw, T1A, Tt, T1B;
+	       T1 = ri[0];
+	       T1z = ii[0];
+	       {
+		    E T2, T3, Tu, Tv;
+		    T2 = ri[WS(is, 5)];
+		    T3 = ri[WS(is, 10)];
+		    T4 = T2 + T3;
+		    T1y = KP866025403 * (T3 - T2);
+		    Tu = ii[WS(is, 5)];
+		    Tv = ii[WS(is, 10)];
+		    Tw = KP866025403 * (Tu - Tv);
+		    T1A = Tu + Tv;
+	       }
+	       T5 = T1 + T4;
+	       T2l = T1z + T1A;
+	       Tt = FNMS(KP500000000, T4, T1);
+	       Tx = Tt - Tw;
+	       TV = Tt + Tw;
+	       T1B = FNMS(KP500000000, T1A, T1z);
+	       T1C = T1y + T1B;
+	       T20 = T1B - T1y;
+	  }
+	  {
+	       E Th, Tk, TJ, T1h, T1i, T1j, TM, T1k, Tm, Tp, TO, T1m, T1n, T1o, TR;
+	       E T1p;
+	       {
+		    E Ti, Tj, TK, TL;
+		    Th = ri[WS(is, 6)];
+		    Ti = ri[WS(is, 11)];
+		    Tj = ri[WS(is, 1)];
+		    Tk = Ti + Tj;
+		    TJ = FNMS(KP500000000, Tk, Th);
+		    T1h = KP866025403 * (Tj - Ti);
+		    T1i = ii[WS(is, 6)];
+		    TK = ii[WS(is, 11)];
+		    TL = ii[WS(is, 1)];
+		    T1j = TK + TL;
+		    TM = KP866025403 * (TK - TL);
+		    T1k = FNMS(KP500000000, T1j, T1i);
+	       }
+	       {
+		    E Tn, To, TP, TQ;
+		    Tm = ri[WS(is, 9)];
+		    Tn = ri[WS(is, 14)];
+		    To = ri[WS(is, 4)];
+		    Tp = Tn + To;
+		    TO = FNMS(KP500000000, Tp, Tm);
+		    T1m = KP866025403 * (To - Tn);
+		    T1n = ii[WS(is, 9)];
+		    TP = ii[WS(is, 14)];
+		    TQ = ii[WS(is, 4)];
+		    T1o = TP + TQ;
+		    TR = KP866025403 * (TP - TQ);
+		    T1p = FNMS(KP500000000, T1o, T1n);
+	       }
+	       Tl = Th + Tk;
+	       Tq = Tm + Tp;
+	       Tr = Tl + Tq;
+	       TN = TJ - TM;
+	       TS = TO - TR;
+	       TT = TN + TS;
+	       T2c = T1i + T1j;
+	       T2d = T1n + T1o;
+	       T2n = T2c + T2d;
+	       T1O = T1k - T1h;
+	       T1P = T1p - T1m;
+	       T22 = T1O + T1P;
+	       T1l = T1h + T1k;
+	       T1q = T1m + T1p;
+	       T1w = T1l + T1q;
+	       TZ = TJ + TM;
+	       T10 = TO + TR;
+	       T11 = TZ + T10;
+	  }
+	  {
+	       E T6, T9, Ty, T16, T17, T18, TB, T19, Tb, Te, TD, T1b, T1c, T1d, TG;
+	       E T1e;
+	       {
+		    E T7, T8, Tz, TA;
+		    T6 = ri[WS(is, 3)];
+		    T7 = ri[WS(is, 8)];
+		    T8 = ri[WS(is, 13)];
+		    T9 = T7 + T8;
+		    Ty = FNMS(KP500000000, T9, T6);
+		    T16 = KP866025403 * (T8 - T7);
+		    T17 = ii[WS(is, 3)];
+		    Tz = ii[WS(is, 8)];
+		    TA = ii[WS(is, 13)];
+		    T18 = Tz + TA;
+		    TB = KP866025403 * (Tz - TA);
+		    T19 = FNMS(KP500000000, T18, T17);
+	       }
+	       {
+		    E Tc, Td, TE, TF;
+		    Tb = ri[WS(is, 12)];
+		    Tc = ri[WS(is, 2)];
+		    Td = ri[WS(is, 7)];
+		    Te = Tc + Td;
+		    TD = FNMS(KP500000000, Te, Tb);
+		    T1b = KP866025403 * (Td - Tc);
+		    T1c = ii[WS(is, 12)];
+		    TE = ii[WS(is, 2)];
+		    TF = ii[WS(is, 7)];
+		    T1d = TE + TF;
+		    TG = KP866025403 * (TE - TF);
+		    T1e = FNMS(KP500000000, T1d, T1c);
+	       }
+	       Ta = T6 + T9;
+	       Tf = Tb + Te;
+	       Tg = Ta + Tf;
+	       TC = Ty - TB;
+	       TH = TD - TG;
+	       TI = TC + TH;
+	       T2f = T17 + T18;
+	       T2g = T1c + T1d;
+	       T2m = T2f + T2g;
+	       T1R = T19 - T16;
+	       T1S = T1e - T1b;
+	       T21 = T1R + T1S;
+	       T1a = T16 + T19;
+	       T1f = T1b + T1e;
+	       T1v = T1a + T1f;
+	       TW = Ty + TB;
+	       TX = TD + TG;
+	       TY = TW + TX;
+	  }
+	  {
+	       E T2a, Ts, T29, T2i, T2k, T2e, T2h, T2j, T2b;
+	       T2a = KP559016994 * (Tg - Tr);
+	       Ts = Tg + Tr;
+	       T29 = FNMS(KP250000000, Ts, T5);
+	       T2e = T2c - T2d;
+	       T2h = T2f - T2g;
+	       T2i = FNMS(KP587785252, T2h, KP951056516 * T2e);
+	       T2k = FMA(KP951056516, T2h, KP587785252 * T2e);
+	       ro[0] = T5 + Ts;
+	       T2j = T2a + T29;
+	       ro[WS(os, 9)] = T2j - T2k;
+	       ro[WS(os, 6)] = T2j + T2k;
+	       T2b = T29 - T2a;
+	       ro[WS(os, 12)] = T2b - T2i;
+	       ro[WS(os, 3)] = T2b + T2i;
+	  }
+	  {
+	       E T2q, T2o, T2p, T2u, T2w, T2s, T2t, T2v, T2r;
+	       T2q = KP559016994 * (T2m - T2n);
+	       T2o = T2m + T2n;
+	       T2p = FNMS(KP250000000, T2o, T2l);
+	       T2s = Tl - Tq;
+	       T2t = Ta - Tf;
+	       T2u = FNMS(KP587785252, T2t, KP951056516 * T2s);
+	       T2w = FMA(KP951056516, T2t, KP587785252 * T2s);
+	       io[0] = T2l + T2o;
+	       T2v = T2q + T2p;
+	       io[WS(os, 6)] = T2v - T2w;
+	       io[WS(os, 9)] = T2w + T2v;
+	       T2r = T2p - T2q;
+	       io[WS(os, 3)] = T2r - T2u;
+	       io[WS(os, 12)] = T2u + T2r;
+	  }
+	  {
+	       E T1M, TU, T1L, T1U, T1W, T1Q, T1T, T1V, T1N;
+	       T1M = KP559016994 * (TI - TT);
+	       TU = TI + TT;
+	       T1L = FNMS(KP250000000, TU, Tx);
+	       T1Q = T1O - T1P;
+	       T1T = T1R - T1S;
+	       T1U = FNMS(KP587785252, T1T, KP951056516 * T1Q);
+	       T1W = FMA(KP951056516, T1T, KP587785252 * T1Q);
+	       ro[WS(os, 5)] = Tx + TU;
+	       T1V = T1M + T1L;
+	       ro[WS(os, 14)] = T1V - T1W;
+	       ro[WS(os, 11)] = T1V + T1W;
+	       T1N = T1L - T1M;
+	       ro[WS(os, 2)] = T1N - T1U;
+	       ro[WS(os, 8)] = T1N + T1U;
+	  }
+	  {
+	       E T25, T23, T24, T1Z, T28, T1X, T1Y, T27, T26;
+	       T25 = KP559016994 * (T21 - T22);
+	       T23 = T21 + T22;
+	       T24 = FNMS(KP250000000, T23, T20);
+	       T1X = TN - TS;
+	       T1Y = TC - TH;
+	       T1Z = FNMS(KP587785252, T1Y, KP951056516 * T1X);
+	       T28 = FMA(KP951056516, T1Y, KP587785252 * T1X);
+	       io[WS(os, 5)] = T20 + T23;
+	       T27 = T25 + T24;
+	       io[WS(os, 11)] = T27 - T28;
+	       io[WS(os, 14)] = T28 + T27;
+	       T26 = T24 - T25;
+	       io[WS(os, 2)] = T1Z + T26;
+	       io[WS(os, 8)] = T26 - T1Z;
+	  }
+	  {
+	       E T1x, T1D, T1E, T1I, T1J, T1G, T1H, T1K, T1F;
+	       T1x = KP559016994 * (T1v - T1w);
+	       T1D = T1v + T1w;
+	       T1E = FNMS(KP250000000, T1D, T1C);
+	       T1G = TW - TX;
+	       T1H = TZ - T10;
+	       T1I = FMA(KP951056516, T1G, KP587785252 * T1H);
+	       T1J = FNMS(KP587785252, T1G, KP951056516 * T1H);
+	       io[WS(os, 10)] = T1C + T1D;
+	       T1K = T1E - T1x;
+	       io[WS(os, 7)] = T1J + T1K;
+	       io[WS(os, 13)] = T1K - T1J;
+	       T1F = T1x + T1E;
+	       io[WS(os, 1)] = T1F - T1I;
+	       io[WS(os, 4)] = T1I + T1F;
+	  }
+	  {
+	       E T13, T12, T14, T1s, T1u, T1g, T1r, T1t, T15;
+	       T13 = KP559016994 * (TY - T11);
+	       T12 = TY + T11;
+	       T14 = FNMS(KP250000000, T12, TV);
+	       T1g = T1a - T1f;
+	       T1r = T1l - T1q;
+	       T1s = FMA(KP951056516, T1g, KP587785252 * T1r);
+	       T1u = FNMS(KP587785252, T1g, KP951056516 * T1r);
+	       ro[WS(os, 10)] = TV + T12;
+	       T1t = T14 - T13;
+	       ro[WS(os, 7)] = T1t - T1u;
+	       ro[WS(os, 13)] = T1t + T1u;
+	       T15 = T13 + T14;
+	       ro[WS(os, 4)] = T15 - T1s;
+	       ro[WS(os, 1)] = T15 + T1s;
+	  }
+     }
+}
+
+static const kdft_desc desc = { 15, "n1_15", {128, 28, 28, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_15) (planner *p) {
+     X(kdft_register) (p, n1_15, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_16.c b/src/fftw3/dft/codelets/standard/n1_16.c
new file mode 100644
index 0000000..6c813a1
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_16.c
@@ -0,0 +1,294 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:37 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 16 -name n1_16 -include n.h */
+
+/*
+ * This function contains 144 FP additions, 24 FP multiplications,
+ * (or, 136 additions, 16 multiplications, 8 fused multiply/add),
+ * 50 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_16(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T7, T1R, T25, TC, TN, T1x, T1H, T1l, Tt, T22, T2h, T1b, T1g, T1E, T1Z;
+	  E T1D, Te, T1S, T26, TJ, TQ, T1m, T1n, TT, Tm, T1X, T2g, T10, T15, T1B;
+	  E T1U, T1A;
+	  {
+	       E T3, TL, Ty, T1k, T6, T1j, TB, TM;
+	       {
+		    E T1, T2, Tw, Tx;
+		    T1 = ri[0];
+		    T2 = ri[WS(is, 8)];
+		    T3 = T1 + T2;
+		    TL = T1 - T2;
+		    Tw = ii[0];
+		    Tx = ii[WS(is, 8)];
+		    Ty = Tw + Tx;
+		    T1k = Tw - Tx;
+	       }
+	       {
+		    E T4, T5, Tz, TA;
+		    T4 = ri[WS(is, 4)];
+		    T5 = ri[WS(is, 12)];
+		    T6 = T4 + T5;
+		    T1j = T4 - T5;
+		    Tz = ii[WS(is, 4)];
+		    TA = ii[WS(is, 12)];
+		    TB = Tz + TA;
+		    TM = Tz - TA;
+	       }
+	       T7 = T3 + T6;
+	       T1R = T3 - T6;
+	       T25 = Ty - TB;
+	       TC = Ty + TB;
+	       TN = TL - TM;
+	       T1x = TL + TM;
+	       T1H = T1k - T1j;
+	       T1l = T1j + T1k;
+	  }
+	  {
+	       E Tp, T17, T1f, T20, Ts, T1c, T1a, T21;
+	       {
+		    E Tn, To, T1d, T1e;
+		    Tn = ri[WS(is, 15)];
+		    To = ri[WS(is, 7)];
+		    Tp = Tn + To;
+		    T17 = Tn - To;
+		    T1d = ii[WS(is, 15)];
+		    T1e = ii[WS(is, 7)];
+		    T1f = T1d - T1e;
+		    T20 = T1d + T1e;
+	       }
+	       {
+		    E Tq, Tr, T18, T19;
+		    Tq = ri[WS(is, 3)];
+		    Tr = ri[WS(is, 11)];
+		    Ts = Tq + Tr;
+		    T1c = Tq - Tr;
+		    T18 = ii[WS(is, 3)];
+		    T19 = ii[WS(is, 11)];
+		    T1a = T18 - T19;
+		    T21 = T18 + T19;
+	       }
+	       Tt = Tp + Ts;
+	       T22 = T20 - T21;
+	       T2h = T20 + T21;
+	       T1b = T17 - T1a;
+	       T1g = T1c + T1f;
+	       T1E = T1f - T1c;
+	       T1Z = Tp - Ts;
+	       T1D = T17 + T1a;
+	  }
+	  {
+	       E Ta, TP, TF, TO, Td, TR, TI, TS;
+	       {
+		    E T8, T9, TD, TE;
+		    T8 = ri[WS(is, 2)];
+		    T9 = ri[WS(is, 10)];
+		    Ta = T8 + T9;
+		    TP = T8 - T9;
+		    TD = ii[WS(is, 2)];
+		    TE = ii[WS(is, 10)];
+		    TF = TD + TE;
+		    TO = TD - TE;
+	       }
+	       {
+		    E Tb, Tc, TG, TH;
+		    Tb = ri[WS(is, 14)];
+		    Tc = ri[WS(is, 6)];
+		    Td = Tb + Tc;
+		    TR = Tb - Tc;
+		    TG = ii[WS(is, 14)];
+		    TH = ii[WS(is, 6)];
+		    TI = TG + TH;
+		    TS = TG - TH;
+	       }
+	       Te = Ta + Td;
+	       T1S = TF - TI;
+	       T26 = Td - Ta;
+	       TJ = TF + TI;
+	       TQ = TO - TP;
+	       T1m = TR - TS;
+	       T1n = TP + TO;
+	       TT = TR + TS;
+	  }
+	  {
+	       E Ti, T11, TZ, T1V, Tl, TW, T14, T1W;
+	       {
+		    E Tg, Th, TX, TY;
+		    Tg = ri[WS(is, 1)];
+		    Th = ri[WS(is, 9)];
+		    Ti = Tg + Th;
+		    T11 = Tg - Th;
+		    TX = ii[WS(is, 1)];
+		    TY = ii[WS(is, 9)];
+		    TZ = TX - TY;
+		    T1V = TX + TY;
+	       }
+	       {
+		    E Tj, Tk, T12, T13;
+		    Tj = ri[WS(is, 5)];
+		    Tk = ri[WS(is, 13)];
+		    Tl = Tj + Tk;
+		    TW = Tj - Tk;
+		    T12 = ii[WS(is, 5)];
+		    T13 = ii[WS(is, 13)];
+		    T14 = T12 - T13;
+		    T1W = T12 + T13;
+	       }
+	       Tm = Ti + Tl;
+	       T1X = T1V - T1W;
+	       T2g = T1V + T1W;
+	       T10 = TW + TZ;
+	       T15 = T11 - T14;
+	       T1B = T11 + T14;
+	       T1U = Ti - Tl;
+	       T1A = TZ - TW;
+	  }
+	  {
+	       E Tf, Tu, T2j, T2k;
+	       Tf = T7 + Te;
+	       Tu = Tm + Tt;
+	       ro[WS(os, 8)] = Tf - Tu;
+	       ro[0] = Tf + Tu;
+	       T2j = TC + TJ;
+	       T2k = T2g + T2h;
+	       io[WS(os, 8)] = T2j - T2k;
+	       io[0] = T2j + T2k;
+	  }
+	  {
+	       E Tv, TK, T2f, T2i;
+	       Tv = Tt - Tm;
+	       TK = TC - TJ;
+	       io[WS(os, 4)] = Tv + TK;
+	       io[WS(os, 12)] = TK - Tv;
+	       T2f = T7 - Te;
+	       T2i = T2g - T2h;
+	       ro[WS(os, 12)] = T2f - T2i;
+	       ro[WS(os, 4)] = T2f + T2i;
+	  }
+	  {
+	       E T1T, T27, T24, T28, T1Y, T23;
+	       T1T = T1R + T1S;
+	       T27 = T25 - T26;
+	       T1Y = T1U + T1X;
+	       T23 = T1Z - T22;
+	       T24 = KP707106781 * (T1Y + T23);
+	       T28 = KP707106781 * (T23 - T1Y);
+	       ro[WS(os, 10)] = T1T - T24;
+	       io[WS(os, 6)] = T27 + T28;
+	       ro[WS(os, 2)] = T1T + T24;
+	       io[WS(os, 14)] = T27 - T28;
+	  }
+	  {
+	       E T29, T2d, T2c, T2e, T2a, T2b;
+	       T29 = T1R - T1S;
+	       T2d = T26 + T25;
+	       T2a = T1X - T1U;
+	       T2b = T1Z + T22;
+	       T2c = KP707106781 * (T2a - T2b);
+	       T2e = KP707106781 * (T2a + T2b);
+	       ro[WS(os, 14)] = T29 - T2c;
+	       io[WS(os, 2)] = T2d + T2e;
+	       ro[WS(os, 6)] = T29 + T2c;
+	       io[WS(os, 10)] = T2d - T2e;
+	  }
+	  {
+	       E TV, T1r, T1p, T1v, T1i, T1q, T1u, T1w, TU, T1o;
+	       TU = KP707106781 * (TQ - TT);
+	       TV = TN + TU;
+	       T1r = TN - TU;
+	       T1o = KP707106781 * (T1m - T1n);
+	       T1p = T1l - T1o;
+	       T1v = T1l + T1o;
+	       {
+		    E T16, T1h, T1s, T1t;
+		    T16 = FMA(KP923879532, T10, KP382683432 * T15);
+		    T1h = FNMS(KP923879532, T1g, KP382683432 * T1b);
+		    T1i = T16 + T1h;
+		    T1q = T1h - T16;
+		    T1s = FNMS(KP923879532, T15, KP382683432 * T10);
+		    T1t = FMA(KP382683432, T1g, KP923879532 * T1b);
+		    T1u = T1s - T1t;
+		    T1w = T1s + T1t;
+	       }
+	       ro[WS(os, 11)] = TV - T1i;
+	       io[WS(os, 11)] = T1v - T1w;
+	       ro[WS(os, 3)] = TV + T1i;
+	       io[WS(os, 3)] = T1v + T1w;
+	       io[WS(os, 15)] = T1p - T1q;
+	       ro[WS(os, 15)] = T1r - T1u;
+	       io[WS(os, 7)] = T1p + T1q;
+	       ro[WS(os, 7)] = T1r + T1u;
+	  }
+	  {
+	       E T1z, T1L, T1J, T1P, T1G, T1K, T1O, T1Q, T1y, T1I;
+	       T1y = KP707106781 * (T1n + T1m);
+	       T1z = T1x + T1y;
+	       T1L = T1x - T1y;
+	       T1I = KP707106781 * (TQ + TT);
+	       T1J = T1H - T1I;
+	       T1P = T1H + T1I;
+	       {
+		    E T1C, T1F, T1M, T1N;
+		    T1C = FMA(KP382683432, T1A, KP923879532 * T1B);
+		    T1F = FNMS(KP382683432, T1E, KP923879532 * T1D);
+		    T1G = T1C + T1F;
+		    T1K = T1F - T1C;
+		    T1M = FNMS(KP382683432, T1B, KP923879532 * T1A);
+		    T1N = FMA(KP923879532, T1E, KP382683432 * T1D);
+		    T1O = T1M - T1N;
+		    T1Q = T1M + T1N;
+	       }
+	       ro[WS(os, 9)] = T1z - T1G;
+	       io[WS(os, 9)] = T1P - T1Q;
+	       ro[WS(os, 1)] = T1z + T1G;
+	       io[WS(os, 1)] = T1P + T1Q;
+	       io[WS(os, 13)] = T1J - T1K;
+	       ro[WS(os, 13)] = T1L - T1O;
+	       io[WS(os, 5)] = T1J + T1K;
+	       ro[WS(os, 5)] = T1L + T1O;
+	  }
+     }
+}
+
+static const kdft_desc desc = { 16, "n1_16", {136, 16, 8, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_16) (planner *p) {
+     X(kdft_register) (p, n1_16, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_2.c b/src/fftw3/dft/codelets/standard/n1_2.c
new file mode 100644
index 0000000..8819b53
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_2.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:31 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 2 -name n1_2 -include n.h */
+
+/*
+ * This function contains 4 FP additions, 0 FP multiplications,
+ * (or, 4 additions, 0 multiplications, 0 fused multiply/add),
+ * 5 stack variables, and 8 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_2.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_2.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_2.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_2(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T2, T3, T4;
+	  T1 = ri[0];
+	  T2 = ri[WS(is, 1)];
+	  ro[WS(os, 1)] = T1 - T2;
+	  ro[0] = T1 + T2;
+	  T3 = ii[0];
+	  T4 = ii[WS(is, 1)];
+	  io[WS(os, 1)] = T3 - T4;
+	  io[0] = T3 + T4;
+     }
+}
+
+static const kdft_desc desc = { 2, "n1_2", {4, 0, 0, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_2) (planner *p) {
+     X(kdft_register) (p, n1_2, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_3.c b/src/fftw3/dft/codelets/standard/n1_3.c
new file mode 100644
index 0000000..f639ac1
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_3.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:32 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 3 -name n1_3 -include n.h */
+
+/*
+ * This function contains 12 FP additions, 4 FP multiplications,
+ * (or, 10 additions, 2 multiplications, 2 fused multiply/add),
+ * 15 stack variables, and 12 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_3.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_3.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_3.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_3(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, Ta, T4, T9, T8, Tb, T5, Tc;
+	  T1 = ri[0];
+	  Ta = ii[0];
+	  {
+	       E T2, T3, T6, T7;
+	       T2 = ri[WS(is, 1)];
+	       T3 = ri[WS(is, 2)];
+	       T4 = T2 + T3;
+	       T9 = KP866025403 * (T3 - T2);
+	       T6 = ii[WS(is, 1)];
+	       T7 = ii[WS(is, 2)];
+	       T8 = KP866025403 * (T6 - T7);
+	       Tb = T6 + T7;
+	  }
+	  ro[0] = T1 + T4;
+	  io[0] = Ta + Tb;
+	  T5 = FNMS(KP500000000, T4, T1);
+	  ro[WS(os, 2)] = T5 - T8;
+	  ro[WS(os, 1)] = T5 + T8;
+	  Tc = FNMS(KP500000000, Tb, Ta);
+	  io[WS(os, 1)] = T9 + Tc;
+	  io[WS(os, 2)] = Tc - T9;
+     }
+}
+
+static const kdft_desc desc = { 3, "n1_3", {10, 2, 2, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_3) (planner *p) {
+     X(kdft_register) (p, n1_3, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_4.c b/src/fftw3/dft/codelets/standard/n1_4.c
new file mode 100644
index 0000000..402046a
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_4.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:32 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 4 -name n1_4 -include n.h */
+
+/*
+ * This function contains 16 FP additions, 0 FP multiplications,
+ * (or, 16 additions, 0 multiplications, 0 fused multiply/add),
+ * 13 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_4.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_4.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_4.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_4(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T3, Tb, T9, Tf, T6, Ta, Te, Tg;
+	  {
+	       E T1, T2, T7, T8;
+	       T1 = ri[0];
+	       T2 = ri[WS(is, 2)];
+	       T3 = T1 + T2;
+	       Tb = T1 - T2;
+	       T7 = ii[0];
+	       T8 = ii[WS(is, 2)];
+	       T9 = T7 - T8;
+	       Tf = T7 + T8;
+	  }
+	  {
+	       E T4, T5, Tc, Td;
+	       T4 = ri[WS(is, 1)];
+	       T5 = ri[WS(is, 3)];
+	       T6 = T4 + T5;
+	       Ta = T4 - T5;
+	       Tc = ii[WS(is, 1)];
+	       Td = ii[WS(is, 3)];
+	       Te = Tc - Td;
+	       Tg = Tc + Td;
+	  }
+	  ro[WS(os, 2)] = T3 - T6;
+	  io[WS(os, 2)] = Tf - Tg;
+	  ro[0] = T3 + T6;
+	  io[0] = Tf + Tg;
+	  io[WS(os, 1)] = T9 - Ta;
+	  ro[WS(os, 1)] = Tb + Te;
+	  io[WS(os, 3)] = Ta + T9;
+	  ro[WS(os, 3)] = Tb - Te;
+     }
+}
+
+static const kdft_desc desc = { 4, "n1_4", {16, 0, 0, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_4) (planner *p) {
+     X(kdft_register) (p, n1_4, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_5.c b/src/fftw3/dft/codelets/standard/n1_5.c
new file mode 100644
index 0000000..8ac5abc
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_5.c
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:32 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 5 -name n1_5 -include n.h */
+
+/*
+ * This function contains 32 FP additions, 12 FP multiplications,
+ * (or, 26 additions, 6 multiplications, 6 fused multiply/add),
+ * 21 stack variables, and 20 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_5.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_5.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_5.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_5(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, To, T8, Tt, T9, Ts, Te, Tp, Th, Tn;
+	  T1 = ri[0];
+	  To = ii[0];
+	  {
+	       E T2, T3, T4, T5, T6, T7;
+	       T2 = ri[WS(is, 1)];
+	       T3 = ri[WS(is, 4)];
+	       T4 = T2 + T3;
+	       T5 = ri[WS(is, 2)];
+	       T6 = ri[WS(is, 3)];
+	       T7 = T5 + T6;
+	       T8 = T4 + T7;
+	       Tt = T5 - T6;
+	       T9 = KP559016994 * (T4 - T7);
+	       Ts = T2 - T3;
+	  }
+	  {
+	       E Tc, Td, Tl, Tf, Tg, Tm;
+	       Tc = ii[WS(is, 1)];
+	       Td = ii[WS(is, 4)];
+	       Tl = Tc + Td;
+	       Tf = ii[WS(is, 2)];
+	       Tg = ii[WS(is, 3)];
+	       Tm = Tf + Tg;
+	       Te = Tc - Td;
+	       Tp = Tl + Tm;
+	       Th = Tf - Tg;
+	       Tn = KP559016994 * (Tl - Tm);
+	  }
+	  ro[0] = T1 + T8;
+	  io[0] = To + Tp;
+	  {
+	       E Ti, Tk, Tb, Tj, Ta;
+	       Ti = FMA(KP951056516, Te, KP587785252 * Th);
+	       Tk = FNMS(KP587785252, Te, KP951056516 * Th);
+	       Ta = FNMS(KP250000000, T8, T1);
+	       Tb = T9 + Ta;
+	       Tj = Ta - T9;
+	       ro[WS(os, 4)] = Tb - Ti;
+	       ro[WS(os, 3)] = Tj + Tk;
+	       ro[WS(os, 1)] = Tb + Ti;
+	       ro[WS(os, 2)] = Tj - Tk;
+	  }
+	  {
+	       E Tu, Tv, Tr, Tw, Tq;
+	       Tu = FMA(KP951056516, Ts, KP587785252 * Tt);
+	       Tv = FNMS(KP587785252, Ts, KP951056516 * Tt);
+	       Tq = FNMS(KP250000000, Tp, To);
+	       Tr = Tn + Tq;
+	       Tw = Tq - Tn;
+	       io[WS(os, 1)] = Tr - Tu;
+	       io[WS(os, 3)] = Tw - Tv;
+	       io[WS(os, 4)] = Tu + Tr;
+	       io[WS(os, 2)] = Tv + Tw;
+	  }
+     }
+}
+
+static const kdft_desc desc = { 5, "n1_5", {26, 6, 6, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_5) (planner *p) {
+     X(kdft_register) (p, n1_5, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_6.c b/src/fftw3/dft/codelets/standard/n1_6.c
new file mode 100644
index 0000000..114679d
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_6.c
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:32 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 6 -name n1_6 -include n.h */
+
+/*
+ * This function contains 36 FP additions, 8 FP multiplications,
+ * (or, 32 additions, 4 multiplications, 4 fused multiply/add),
+ * 23 stack variables, and 24 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_6.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_6.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_6.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_6(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T3, Tb, Tq, Tx, T6, Tc, T9, Td, Ta, Te, Ti, Tu, Tl, Tv, Tr;
+	  E Ty;
+	  {
+	       E T1, T2, To, Tp;
+	       T1 = ri[0];
+	       T2 = ri[WS(is, 3)];
+	       T3 = T1 - T2;
+	       Tb = T1 + T2;
+	       To = ii[0];
+	       Tp = ii[WS(is, 3)];
+	       Tq = To - Tp;
+	       Tx = To + Tp;
+	  }
+	  {
+	       E T4, T5, T7, T8;
+	       T4 = ri[WS(is, 2)];
+	       T5 = ri[WS(is, 5)];
+	       T6 = T4 - T5;
+	       Tc = T4 + T5;
+	       T7 = ri[WS(is, 4)];
+	       T8 = ri[WS(is, 1)];
+	       T9 = T7 - T8;
+	       Td = T7 + T8;
+	  }
+	  Ta = T6 + T9;
+	  Te = Tc + Td;
+	  {
+	       E Tg, Th, Tj, Tk;
+	       Tg = ii[WS(is, 2)];
+	       Th = ii[WS(is, 5)];
+	       Ti = Tg - Th;
+	       Tu = Tg + Th;
+	       Tj = ii[WS(is, 4)];
+	       Tk = ii[WS(is, 1)];
+	       Tl = Tj - Tk;
+	       Tv = Tj + Tk;
+	  }
+	  Tr = Ti + Tl;
+	  Ty = Tu + Tv;
+	  ro[WS(os, 3)] = T3 + Ta;
+	  io[WS(os, 3)] = Tq + Tr;
+	  ro[0] = Tb + Te;
+	  io[0] = Tx + Ty;
+	  {
+	       E Tf, Tm, Tn, Ts;
+	       Tf = FNMS(KP500000000, Ta, T3);
+	       Tm = KP866025403 * (Ti - Tl);
+	       ro[WS(os, 5)] = Tf - Tm;
+	       ro[WS(os, 1)] = Tf + Tm;
+	       Tn = KP866025403 * (T9 - T6);
+	       Ts = FNMS(KP500000000, Tr, Tq);
+	       io[WS(os, 1)] = Tn + Ts;
+	       io[WS(os, 5)] = Ts - Tn;
+	  }
+	  {
+	       E Tt, Tw, Tz, TA;
+	       Tt = FNMS(KP500000000, Te, Tb);
+	       Tw = KP866025403 * (Tu - Tv);
+	       ro[WS(os, 2)] = Tt - Tw;
+	       ro[WS(os, 4)] = Tt + Tw;
+	       Tz = FNMS(KP500000000, Ty, Tx);
+	       TA = KP866025403 * (Td - Tc);
+	       io[WS(os, 2)] = Tz - TA;
+	       io[WS(os, 4)] = TA + Tz;
+	  }
+     }
+}
+
+static const kdft_desc desc = { 6, "n1_6", {32, 4, 4, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_6) (planner *p) {
+     X(kdft_register) (p, n1_6, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_7.c b/src/fftw3/dft/codelets/standard/n1_7.c
new file mode 100644
index 0000000..03c892d
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_7.c
@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:32 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 7 -name n1_7 -include n.h */
+
+/*
+ * This function contains 60 FP additions, 36 FP multiplications,
+ * (or, 36 additions, 12 multiplications, 24 fused multiply/add),
+ * 25 stack variables, and 28 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_7.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_7.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_7.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_7(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP222520933, +0.222520933956314404288902564496794759466355569);
+     DK(KP900968867, +0.900968867902419126236102319507445051165919162);
+     DK(KP623489801, +0.623489801858733530525004884004239810632274731);
+     DK(KP433883739, +0.433883739117558120475768332848358754609990728);
+     DK(KP781831482, +0.781831482468029808708444526674057750232334519);
+     DK(KP974927912, +0.974927912181823607018131682993931217232785801);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, Tu, T4, Tq, Te, Tx, T7, Ts, Tk, Tv, Ta, Tr, Th, Tw;
+	  T1 = ri[0];
+	  Tu = ii[0];
+	  {
+	       E T2, T3, Tc, Td;
+	       T2 = ri[WS(is, 1)];
+	       T3 = ri[WS(is, 6)];
+	       T4 = T2 + T3;
+	       Tq = T3 - T2;
+	       Tc = ii[WS(is, 1)];
+	       Td = ii[WS(is, 6)];
+	       Te = Tc - Td;
+	       Tx = Tc + Td;
+	  }
+	  {
+	       E T5, T6, Ti, Tj;
+	       T5 = ri[WS(is, 2)];
+	       T6 = ri[WS(is, 5)];
+	       T7 = T5 + T6;
+	       Ts = T6 - T5;
+	       Ti = ii[WS(is, 2)];
+	       Tj = ii[WS(is, 5)];
+	       Tk = Ti - Tj;
+	       Tv = Ti + Tj;
+	  }
+	  {
+	       E T8, T9, Tf, Tg;
+	       T8 = ri[WS(is, 3)];
+	       T9 = ri[WS(is, 4)];
+	       Ta = T8 + T9;
+	       Tr = T9 - T8;
+	       Tf = ii[WS(is, 3)];
+	       Tg = ii[WS(is, 4)];
+	       Th = Tf - Tg;
+	       Tw = Tf + Tg;
+	  }
+	  ro[0] = T1 + T4 + T7 + Ta;
+	  io[0] = Tu + Tx + Tv + Tw;
+	  {
+	       E Tl, Tb, TB, TC;
+	       Tl = FNMS(KP781831482, Th, KP974927912 * Te) - (KP433883739 * Tk);
+	       Tb = FMA(KP623489801, Ta, T1) + FNMA(KP900968867, T7, KP222520933 * T4);
+	       ro[WS(os, 5)] = Tb - Tl;
+	       ro[WS(os, 2)] = Tb + Tl;
+	       TB = FNMS(KP781831482, Tr, KP974927912 * Tq) - (KP433883739 * Ts);
+	       TC = FMA(KP623489801, Tw, Tu) + FNMA(KP900968867, Tv, KP222520933 * Tx);
+	       io[WS(os, 2)] = TB + TC;
+	       io[WS(os, 5)] = TC - TB;
+	  }
+	  {
+	       E Tn, Tm, Tz, TA;
+	       Tn = FMA(KP781831482, Te, KP974927912 * Tk) + (KP433883739 * Th);
+	       Tm = FMA(KP623489801, T4, T1) + FNMA(KP900968867, Ta, KP222520933 * T7);
+	       ro[WS(os, 6)] = Tm - Tn;
+	       ro[WS(os, 1)] = Tm + Tn;
+	       Tz = FMA(KP781831482, Tq, KP974927912 * Ts) + (KP433883739 * Tr);
+	       TA = FMA(KP623489801, Tx, Tu) + FNMA(KP900968867, Tw, KP222520933 * Tv);
+	       io[WS(os, 1)] = Tz + TA;
+	       io[WS(os, 6)] = TA - Tz;
+	  }
+	  {
+	       E Tp, To, Tt, Ty;
+	       Tp = FMA(KP433883739, Te, KP974927912 * Th) - (KP781831482 * Tk);
+	       To = FMA(KP623489801, T7, T1) + FNMA(KP222520933, Ta, KP900968867 * T4);
+	       ro[WS(os, 4)] = To - Tp;
+	       ro[WS(os, 3)] = To + Tp;
+	       Tt = FMA(KP433883739, Tq, KP974927912 * Tr) - (KP781831482 * Ts);
+	       Ty = FMA(KP623489801, Tv, Tu) + FNMA(KP222520933, Tw, KP900968867 * Tx);
+	       io[WS(os, 3)] = Tt + Ty;
+	       io[WS(os, 4)] = Ty - Tt;
+	  }
+     }
+}
+
+static const kdft_desc desc = { 7, "n1_7", {36, 12, 24, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_7) (planner *p) {
+     X(kdft_register) (p, n1_7, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_8.c b/src/fftw3/dft/codelets/standard/n1_8.c
new file mode 100644
index 0000000..114592d
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_8.c
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:32 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 8 -name n1_8 -include n.h */
+
+/*
+ * This function contains 52 FP additions, 4 FP multiplications,
+ * (or, 52 additions, 4 multiplications, 0 fused multiply/add),
+ * 28 stack variables, and 32 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_8.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_8.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_8.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_8(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T3, Tn, Ti, TC, T6, TB, Tl, To, Td, TN, Tz, TH, Ta, TM, Tu;
+	  E TG;
+	  {
+	       E T1, T2, Tj, Tk;
+	       T1 = ri[0];
+	       T2 = ri[WS(is, 4)];
+	       T3 = T1 + T2;
+	       Tn = T1 - T2;
+	       {
+		    E Tg, Th, T4, T5;
+		    Tg = ii[0];
+		    Th = ii[WS(is, 4)];
+		    Ti = Tg + Th;
+		    TC = Tg - Th;
+		    T4 = ri[WS(is, 2)];
+		    T5 = ri[WS(is, 6)];
+		    T6 = T4 + T5;
+		    TB = T4 - T5;
+	       }
+	       Tj = ii[WS(is, 2)];
+	       Tk = ii[WS(is, 6)];
+	       Tl = Tj + Tk;
+	       To = Tj - Tk;
+	       {
+		    E Tb, Tc, Tv, Tw, Tx, Ty;
+		    Tb = ri[WS(is, 7)];
+		    Tc = ri[WS(is, 3)];
+		    Tv = Tb - Tc;
+		    Tw = ii[WS(is, 7)];
+		    Tx = ii[WS(is, 3)];
+		    Ty = Tw - Tx;
+		    Td = Tb + Tc;
+		    TN = Tw + Tx;
+		    Tz = Tv - Ty;
+		    TH = Tv + Ty;
+	       }
+	       {
+		    E T8, T9, Tq, Tr, Ts, Tt;
+		    T8 = ri[WS(is, 1)];
+		    T9 = ri[WS(is, 5)];
+		    Tq = T8 - T9;
+		    Tr = ii[WS(is, 1)];
+		    Ts = ii[WS(is, 5)];
+		    Tt = Tr - Ts;
+		    Ta = T8 + T9;
+		    TM = Tr + Ts;
+		    Tu = Tq + Tt;
+		    TG = Tt - Tq;
+	       }
+	  }
+	  {
+	       E T7, Te, TP, TQ;
+	       T7 = T3 + T6;
+	       Te = Ta + Td;
+	       ro[WS(os, 4)] = T7 - Te;
+	       ro[0] = T7 + Te;
+	       TP = Ti + Tl;
+	       TQ = TM + TN;
+	       io[WS(os, 4)] = TP - TQ;
+	       io[0] = TP + TQ;
+	  }
+	  {
+	       E Tf, Tm, TL, TO;
+	       Tf = Td - Ta;
+	       Tm = Ti - Tl;
+	       io[WS(os, 2)] = Tf + Tm;
+	       io[WS(os, 6)] = Tm - Tf;
+	       TL = T3 - T6;
+	       TO = TM - TN;
+	       ro[WS(os, 6)] = TL - TO;
+	       ro[WS(os, 2)] = TL + TO;
+	  }
+	  {
+	       E Tp, TA, TJ, TK;
+	       Tp = Tn + To;
+	       TA = KP707106781 * (Tu + Tz);
+	       ro[WS(os, 5)] = Tp - TA;
+	       ro[WS(os, 1)] = Tp + TA;
+	       TJ = TC - TB;
+	       TK = KP707106781 * (TG + TH);
+	       io[WS(os, 5)] = TJ - TK;
+	       io[WS(os, 1)] = TJ + TK;
+	  }
+	  {
+	       E TD, TE, TF, TI;
+	       TD = TB + TC;
+	       TE = KP707106781 * (Tz - Tu);
+	       io[WS(os, 7)] = TD - TE;
+	       io[WS(os, 3)] = TD + TE;
+	       TF = Tn - To;
+	       TI = KP707106781 * (TG - TH);
+	       ro[WS(os, 7)] = TF - TI;
+	       ro[WS(os, 3)] = TF + TI;
+	  }
+     }
+}
+
+static const kdft_desc desc = { 8, "n1_8", {52, 4, 0, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_8) (planner *p) {
+     X(kdft_register) (p, n1_8, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/n1_9.c b/src/fftw3/dft/codelets/standard/n1_9.c
new file mode 100644
index 0000000..07308dc
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/n1_9.c
@@ -0,0 +1,193 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:32 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_notw -compact -variables 4 -n 9 -name n1_9 -include n.h */
+
+/*
+ * This function contains 80 FP additions, 40 FP multiplications,
+ * (or, 60 additions, 20 multiplications, 20 fused multiply/add),
+ * 39 stack variables, and 36 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: n1_9.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_9.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: n1_9.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "n.h"
+
+static void n1_9(const R *ri, const R *ii, R *ro, R *io, stride is, stride os, int v, int ivs, int ovs)
+{
+     DK(KP939692620, +0.939692620785908384054109277324731469936208134);
+     DK(KP342020143, +0.342020143325668733044099614682259580763083368);
+     DK(KP984807753, +0.984807753012208059366743024589523013670643252);
+     DK(KP173648177, +0.173648177666930348851716626769314796000375677);
+     DK(KP642787609, +0.642787609686539326322643409907263432907559884);
+     DK(KP766044443, +0.766044443118978035202392650555416673935832457);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T5, TO, Th, Tk, T1g, TR, Ta, T1c, Tq, TW, Tv, TX, Tf, T1d, TB;
+	  E T10, TG, TZ;
+	  {
+	       E T1, T2, T3, T4;
+	       T1 = ri[0];
+	       T2 = ri[WS(is, 3)];
+	       T3 = ri[WS(is, 6)];
+	       T4 = T2 + T3;
+	       T5 = T1 + T4;
+	       TO = KP866025403 * (T3 - T2);
+	       Th = FNMS(KP500000000, T4, T1);
+	  }
+	  {
+	       E TP, Ti, Tj, TQ;
+	       TP = ii[0];
+	       Ti = ii[WS(is, 3)];
+	       Tj = ii[WS(is, 6)];
+	       TQ = Ti + Tj;
+	       Tk = KP866025403 * (Ti - Tj);
+	       T1g = TP + TQ;
+	       TR = FNMS(KP500000000, TQ, TP);
+	  }
+	  {
+	       E T6, Ts, T9, Tr, Tp, Tt, Tm, Tu;
+	       T6 = ri[WS(is, 1)];
+	       Ts = ii[WS(is, 1)];
+	       {
+		    E T7, T8, Tn, To;
+		    T7 = ri[WS(is, 4)];
+		    T8 = ri[WS(is, 7)];
+		    T9 = T7 + T8;
+		    Tr = KP866025403 * (T8 - T7);
+		    Tn = ii[WS(is, 4)];
+		    To = ii[WS(is, 7)];
+		    Tp = KP866025403 * (Tn - To);
+		    Tt = Tn + To;
+	       }
+	       Ta = T6 + T9;
+	       T1c = Ts + Tt;
+	       Tm = FNMS(KP500000000, T9, T6);
+	       Tq = Tm + Tp;
+	       TW = Tm - Tp;
+	       Tu = FNMS(KP500000000, Tt, Ts);
+	       Tv = Tr + Tu;
+	       TX = Tu - Tr;
+	  }
+	  {
+	       E Tb, TD, Te, TC, TA, TE, Tx, TF;
+	       Tb = ri[WS(is, 2)];
+	       TD = ii[WS(is, 2)];
+	       {
+		    E Tc, Td, Ty, Tz;
+		    Tc = ri[WS(is, 5)];
+		    Td = ri[WS(is, 8)];
+		    Te = Tc + Td;
+		    TC = KP866025403 * (Td - Tc);
+		    Ty = ii[WS(is, 5)];
+		    Tz = ii[WS(is, 8)];
+		    TA = KP866025403 * (Ty - Tz);
+		    TE = Ty + Tz;
+	       }
+	       Tf = Tb + Te;
+	       T1d = TD + TE;
+	       Tx = FNMS(KP500000000, Te, Tb);
+	       TB = Tx + TA;
+	       T10 = Tx - TA;
+	       TF = FNMS(KP500000000, TE, TD);
+	       TG = TC + TF;
+	       TZ = TF - TC;
+	  }
+	  {
+	       E T1e, Tg, T1b, T1f, T1h, T1i;
+	       T1e = KP866025403 * (T1c - T1d);
+	       Tg = Ta + Tf;
+	       T1b = FNMS(KP500000000, Tg, T5);
+	       ro[0] = T5 + Tg;
+	       ro[WS(os, 3)] = T1b + T1e;
+	       ro[WS(os, 6)] = T1b - T1e;
+	       T1f = KP866025403 * (Tf - Ta);
+	       T1h = T1c + T1d;
+	       T1i = FNMS(KP500000000, T1h, T1g);
+	       io[WS(os, 3)] = T1f + T1i;
+	       io[0] = T1g + T1h;
+	       io[WS(os, 6)] = T1i - T1f;
+	  }
+	  {
+	       E Tl, TS, TI, TN, TM, TT, TJ, TU;
+	       Tl = Th + Tk;
+	       TS = TO + TR;
+	       {
+		    E Tw, TH, TK, TL;
+		    Tw = FMA(KP766044443, Tq, KP642787609 * Tv);
+		    TH = FMA(KP173648177, TB, KP984807753 * TG);
+		    TI = Tw + TH;
+		    TN = KP866025403 * (TH - Tw);
+		    TK = FNMS(KP642787609, Tq, KP766044443 * Tv);
+		    TL = FNMS(KP984807753, TB, KP173648177 * TG);
+		    TM = KP866025403 * (TK - TL);
+		    TT = TK + TL;
+	       }
+	       ro[WS(os, 1)] = Tl + TI;
+	       io[WS(os, 1)] = TS + TT;
+	       TJ = FNMS(KP500000000, TI, Tl);
+	       ro[WS(os, 7)] = TJ - TM;
+	       ro[WS(os, 4)] = TJ + TM;
+	       TU = FNMS(KP500000000, TT, TS);
+	       io[WS(os, 4)] = TN + TU;
+	       io[WS(os, 7)] = TU - TN;
+	  }
+	  {
+	       E TV, T14, T12, T13, T17, T1a, T18, T19;
+	       TV = Th - Tk;
+	       T14 = TR - TO;
+	       {
+		    E TY, T11, T15, T16;
+		    TY = FMA(KP173648177, TW, KP984807753 * TX);
+		    T11 = FNMS(KP939692620, T10, KP342020143 * TZ);
+		    T12 = TY + T11;
+		    T13 = KP866025403 * (T11 - TY);
+		    T15 = FNMS(KP984807753, TW, KP173648177 * TX);
+		    T16 = FMA(KP342020143, T10, KP939692620 * TZ);
+		    T17 = T15 - T16;
+		    T1a = KP866025403 * (T15 + T16);
+	       }
+	       ro[WS(os, 2)] = TV + T12;
+	       io[WS(os, 2)] = T14 + T17;
+	       T18 = FNMS(KP500000000, T17, T14);
+	       io[WS(os, 5)] = T13 + T18;
+	       io[WS(os, 8)] = T18 - T13;
+	       T19 = FNMS(KP500000000, T12, TV);
+	       ro[WS(os, 8)] = T19 - T1a;
+	       ro[WS(os, 5)] = T19 + T1a;
+	  }
+     }
+}
+
+static const kdft_desc desc = { 9, "n1_9", {60, 20, 20, 0}, &GENUS, 0, 0, 0, 0 };
+void X(codelet_n1_9) (planner *p) {
+     X(kdft_register) (p, n1_9, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/scodlist.c b/src/fftw3/dft/codelets/standard/scodlist.c
new file mode 100644
index 0000000..9524d36
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/scodlist.c
@@ -0,0 +1,82 @@
+#include "ifftw.h"
+
+extern void X(codelet_n1_2)(planner *);
+extern void X(codelet_n1_3)(planner *);
+extern void X(codelet_n1_4)(planner *);
+extern void X(codelet_n1_5)(planner *);
+extern void X(codelet_n1_6)(planner *);
+extern void X(codelet_n1_7)(planner *);
+extern void X(codelet_n1_8)(planner *);
+extern void X(codelet_n1_9)(planner *);
+extern void X(codelet_n1_10)(planner *);
+extern void X(codelet_n1_11)(planner *);
+extern void X(codelet_n1_12)(planner *);
+extern void X(codelet_n1_13)(planner *);
+extern void X(codelet_n1_14)(planner *);
+extern void X(codelet_n1_15)(planner *);
+extern void X(codelet_n1_16)(planner *);
+extern void X(codelet_m1_16)(planner *);
+extern void X(codelet_m1_32)(planner *);
+extern void X(codelet_m1_64)(planner *);
+extern void X(codelet_t1_2)(planner *);
+extern void X(codelet_t1_3)(planner *);
+extern void X(codelet_t1_4)(planner *);
+extern void X(codelet_t1_5)(planner *);
+extern void X(codelet_t1_6)(planner *);
+extern void X(codelet_t1_7)(planner *);
+extern void X(codelet_t1_8)(planner *);
+extern void X(codelet_t1_9)(planner *);
+extern void X(codelet_t1_10)(planner *);
+extern void X(codelet_t1_12)(planner *);
+extern void X(codelet_t1_15)(planner *);
+extern void X(codelet_t1_16)(planner *);
+extern void X(codelet_t1_32)(planner *);
+extern void X(codelet_t1_64)(planner *);
+extern void X(codelet_t2_4)(planner *);
+extern void X(codelet_t2_8)(planner *);
+extern void X(codelet_t2_16)(planner *);
+extern void X(codelet_t2_32)(planner *);
+extern void X(codelet_t2_64)(planner *);
+
+
+extern const solvtab X(solvtab_dft_standard);
+const solvtab X(solvtab_dft_standard) = {
+   SOLVTAB(X(codelet_n1_2)),
+   SOLVTAB(X(codelet_n1_3)),
+   SOLVTAB(X(codelet_n1_4)),
+   SOLVTAB(X(codelet_n1_5)),
+   SOLVTAB(X(codelet_n1_6)),
+   SOLVTAB(X(codelet_n1_7)),
+   SOLVTAB(X(codelet_n1_8)),
+   SOLVTAB(X(codelet_n1_9)),
+   SOLVTAB(X(codelet_n1_10)),
+   SOLVTAB(X(codelet_n1_11)),
+   SOLVTAB(X(codelet_n1_12)),
+   SOLVTAB(X(codelet_n1_13)),
+   SOLVTAB(X(codelet_n1_14)),
+   SOLVTAB(X(codelet_n1_15)),
+   SOLVTAB(X(codelet_n1_16)),
+   SOLVTAB(X(codelet_m1_16)),
+   SOLVTAB(X(codelet_m1_32)),
+   SOLVTAB(X(codelet_m1_64)),
+   SOLVTAB(X(codelet_t1_2)),
+   SOLVTAB(X(codelet_t1_3)),
+   SOLVTAB(X(codelet_t1_4)),
+   SOLVTAB(X(codelet_t1_5)),
+   SOLVTAB(X(codelet_t1_6)),
+   SOLVTAB(X(codelet_t1_7)),
+   SOLVTAB(X(codelet_t1_8)),
+   SOLVTAB(X(codelet_t1_9)),
+   SOLVTAB(X(codelet_t1_10)),
+   SOLVTAB(X(codelet_t1_12)),
+   SOLVTAB(X(codelet_t1_15)),
+   SOLVTAB(X(codelet_t1_16)),
+   SOLVTAB(X(codelet_t1_32)),
+   SOLVTAB(X(codelet_t1_64)),
+   SOLVTAB(X(codelet_t2_4)),
+   SOLVTAB(X(codelet_t2_8)),
+   SOLVTAB(X(codelet_t2_16)),
+   SOLVTAB(X(codelet_t2_32)),
+   SOLVTAB(X(codelet_t2_64)),
+   SOLVTAB_END
+};
diff --git a/src/fftw3/dft/codelets/standard/t1_10.c b/src/fftw3/dft/codelets/standard/t1_10.c
new file mode 100644
index 0000000..cd11522
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_10.c
@@ -0,0 +1,253 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:00 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 10 -name t1_10 -include t.h */
+
+/*
+ * This function contains 102 FP additions, 60 FP multiplications,
+ * (or, 72 additions, 30 multiplications, 30 fused multiply/add),
+ * 45 stack variables, and 40 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_10.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_10.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_10.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_10(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 18) {
+	  E T7, T1O, TT, T1C, TF, TQ, TR, T1o, T1p, T1y, TX, TY, TZ, T1d, T1g;
+	  E T1M, Ti, Tt, Tu, T1r, T1s, T1x, TU, TV, TW, T16, T19, T1L;
+	  {
+	       E T1, T1B, T6, T1A;
+	       T1 = ri[0];
+	       T1B = ii[0];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = ri[WS(ios, 5)];
+		    T5 = ii[WS(ios, 5)];
+		    T2 = W[8];
+		    T4 = W[9];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T1A = FNMS(T4, T3, T2 * T5);
+	       }
+	       T7 = T1 - T6;
+	       T1O = T1B - T1A;
+	       TT = T1 + T6;
+	       T1C = T1A + T1B;
+	  }
+	  {
+	       E Tz, T1b, TP, T1f, TE, T1c, TK, T1e;
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = ri[WS(ios, 4)];
+		    Ty = ii[WS(ios, 4)];
+		    Tv = W[6];
+		    Tx = W[7];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T1b = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E TM, TO, TL, TN;
+		    TM = ri[WS(ios, 1)];
+		    TO = ii[WS(ios, 1)];
+		    TL = W[0];
+		    TN = W[1];
+		    TP = FMA(TL, TM, TN * TO);
+		    T1f = FNMS(TN, TM, TL * TO);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = ri[WS(ios, 9)];
+		    TD = ii[WS(ios, 9)];
+		    TA = W[16];
+		    TC = W[17];
+		    TE = FMA(TA, TB, TC * TD);
+		    T1c = FNMS(TC, TB, TA * TD);
+	       }
+	       {
+		    E TH, TJ, TG, TI;
+		    TH = ri[WS(ios, 6)];
+		    TJ = ii[WS(ios, 6)];
+		    TG = W[10];
+		    TI = W[11];
+		    TK = FMA(TG, TH, TI * TJ);
+		    T1e = FNMS(TI, TH, TG * TJ);
+	       }
+	       TF = Tz - TE;
+	       TQ = TK - TP;
+	       TR = TF + TQ;
+	       T1o = T1b + T1c;
+	       T1p = T1e + T1f;
+	       T1y = T1o + T1p;
+	       TX = Tz + TE;
+	       TY = TK + TP;
+	       TZ = TX + TY;
+	       T1d = T1b - T1c;
+	       T1g = T1e - T1f;
+	       T1M = T1d + T1g;
+	  }
+	  {
+	       E Tc, T14, Ts, T18, Th, T15, Tn, T17;
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = ri[WS(ios, 2)];
+		    Tb = ii[WS(ios, 2)];
+		    T8 = W[2];
+		    Ta = W[3];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    T14 = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = ri[WS(ios, 3)];
+		    Tr = ii[WS(ios, 3)];
+		    To = W[4];
+		    Tq = W[5];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    T18 = FNMS(Tq, Tp, To * Tr);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = ri[WS(ios, 7)];
+		    Tg = ii[WS(ios, 7)];
+		    Td = W[12];
+		    Tf = W[13];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    T15 = FNMS(Tf, Te, Td * Tg);
+	       }
+	       {
+		    E Tk, Tm, Tj, Tl;
+		    Tk = ri[WS(ios, 8)];
+		    Tm = ii[WS(ios, 8)];
+		    Tj = W[14];
+		    Tl = W[15];
+		    Tn = FMA(Tj, Tk, Tl * Tm);
+		    T17 = FNMS(Tl, Tk, Tj * Tm);
+	       }
+	       Ti = Tc - Th;
+	       Tt = Tn - Ts;
+	       Tu = Ti + Tt;
+	       T1r = T14 + T15;
+	       T1s = T17 + T18;
+	       T1x = T1r + T1s;
+	       TU = Tc + Th;
+	       TV = Tn + Ts;
+	       TW = TU + TV;
+	       T16 = T14 - T15;
+	       T19 = T17 - T18;
+	       T1L = T16 + T19;
+	  }
+	  {
+	       E T11, TS, T12, T1i, T1k, T1a, T1h, T1j, T13;
+	       T11 = KP559016994 * (Tu - TR);
+	       TS = Tu + TR;
+	       T12 = FNMS(KP250000000, TS, T7);
+	       T1a = T16 - T19;
+	       T1h = T1d - T1g;
+	       T1i = FMA(KP951056516, T1a, KP587785252 * T1h);
+	       T1k = FNMS(KP587785252, T1a, KP951056516 * T1h);
+	       ri[WS(ios, 5)] = T7 + TS;
+	       T1j = T12 - T11;
+	       ri[WS(ios, 7)] = T1j - T1k;
+	       ri[WS(ios, 3)] = T1j + T1k;
+	       T13 = T11 + T12;
+	       ri[WS(ios, 9)] = T13 - T1i;
+	       ri[WS(ios, 1)] = T13 + T1i;
+	  }
+	  {
+	       E T1N, T1P, T1Q, T1U, T1W, T1S, T1T, T1V, T1R;
+	       T1N = KP559016994 * (T1L - T1M);
+	       T1P = T1L + T1M;
+	       T1Q = FNMS(KP250000000, T1P, T1O);
+	       T1S = Ti - Tt;
+	       T1T = TF - TQ;
+	       T1U = FMA(KP951056516, T1S, KP587785252 * T1T);
+	       T1W = FNMS(KP587785252, T1S, KP951056516 * T1T);
+	       ii[WS(ios, 5)] = T1P + T1O;
+	       T1V = T1Q - T1N;
+	       ii[WS(ios, 3)] = T1V - T1W;
+	       ii[WS(ios, 7)] = T1W + T1V;
+	       T1R = T1N + T1Q;
+	       ii[WS(ios, 1)] = T1R - T1U;
+	       ii[WS(ios, 9)] = T1U + T1R;
+	  }
+	  {
+	       E T1m, T10, T1l, T1u, T1w, T1q, T1t, T1v, T1n;
+	       T1m = KP559016994 * (TW - TZ);
+	       T10 = TW + TZ;
+	       T1l = FNMS(KP250000000, T10, TT);
+	       T1q = T1o - T1p;
+	       T1t = T1r - T1s;
+	       T1u = FNMS(KP587785252, T1t, KP951056516 * T1q);
+	       T1w = FMA(KP951056516, T1t, KP587785252 * T1q);
+	       ri[0] = TT + T10;
+	       T1v = T1m + T1l;
+	       ri[WS(ios, 4)] = T1v - T1w;
+	       ri[WS(ios, 6)] = T1v + T1w;
+	       T1n = T1l - T1m;
+	       ri[WS(ios, 2)] = T1n - T1u;
+	       ri[WS(ios, 8)] = T1n + T1u;
+	  }
+	  {
+	       E T1H, T1z, T1G, T1F, T1J, T1D, T1E, T1K, T1I;
+	       T1H = KP559016994 * (T1x - T1y);
+	       T1z = T1x + T1y;
+	       T1G = FNMS(KP250000000, T1z, T1C);
+	       T1D = TX - TY;
+	       T1E = TU - TV;
+	       T1F = FNMS(KP587785252, T1E, KP951056516 * T1D);
+	       T1J = FMA(KP951056516, T1E, KP587785252 * T1D);
+	       ii[0] = T1z + T1C;
+	       T1K = T1H + T1G;
+	       ii[WS(ios, 4)] = T1J + T1K;
+	       ii[WS(ios, 6)] = T1K - T1J;
+	       T1I = T1G - T1H;
+	       ii[WS(ios, 2)] = T1F + T1I;
+	       ii[WS(ios, 8)] = T1I - T1F;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 10},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 10, "t1_10", twinstr, {72, 30, 30, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_10) (planner *p) {
+     X(kdft_dit_register) (p, t1_10, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_12.c b/src/fftw3/dft/codelets/standard/t1_12.c
new file mode 100644
index 0000000..e0ea2eb
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_12.c
@@ -0,0 +1,300 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:03 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 12 -name t1_12 -include t.h */
+
+/*
+ * This function contains 118 FP additions, 60 FP multiplications,
+ * (or, 88 additions, 30 multiplications, 30 fused multiply/add),
+ * 47 stack variables, and 48 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_12.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_12.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_12.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_12(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 22) {
+	  E T1, T1W, T18, T21, Tc, T15, T1V, T22, TR, T1E, T1o, T1D, T12, T1l, T1F;
+	  E T1G, Ti, T1S, T1d, T24, Tt, T1a, T1T, T25, TA, T1z, T1j, T1y, TL, T1g;
+	  E T1A, T1B;
+	  {
+	       E T6, T16, Tb, T17;
+	       T1 = ri[0];
+	       T1W = ii[0];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = ri[WS(ios, 4)];
+		    T5 = ii[WS(ios, 4)];
+		    T2 = W[6];
+		    T4 = W[7];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T16 = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T8, Ta, T7, T9;
+		    T8 = ri[WS(ios, 8)];
+		    Ta = ii[WS(ios, 8)];
+		    T7 = W[14];
+		    T9 = W[15];
+		    Tb = FMA(T7, T8, T9 * Ta);
+		    T17 = FNMS(T9, T8, T7 * Ta);
+	       }
+	       T18 = KP866025403 * (T16 - T17);
+	       T21 = KP866025403 * (Tb - T6);
+	       Tc = T6 + Tb;
+	       T15 = FNMS(KP500000000, Tc, T1);
+	       T1V = T16 + T17;
+	       T22 = FNMS(KP500000000, T1V, T1W);
+	  }
+	  {
+	       E T11, T1n, TW, T1m;
+	       {
+		    E TO, TQ, TN, TP;
+		    TO = ri[WS(ios, 9)];
+		    TQ = ii[WS(ios, 9)];
+		    TN = W[16];
+		    TP = W[17];
+		    TR = FMA(TN, TO, TP * TQ);
+		    T1E = FNMS(TP, TO, TN * TQ);
+	       }
+	       {
+		    E TY, T10, TX, TZ;
+		    TY = ri[WS(ios, 5)];
+		    T10 = ii[WS(ios, 5)];
+		    TX = W[8];
+		    TZ = W[9];
+		    T11 = FMA(TX, TY, TZ * T10);
+		    T1n = FNMS(TZ, TY, TX * T10);
+	       }
+	       {
+		    E TT, TV, TS, TU;
+		    TT = ri[WS(ios, 1)];
+		    TV = ii[WS(ios, 1)];
+		    TS = W[0];
+		    TU = W[1];
+		    TW = FMA(TS, TT, TU * TV);
+		    T1m = FNMS(TU, TT, TS * TV);
+	       }
+	       T1o = KP866025403 * (T1m - T1n);
+	       T1D = KP866025403 * (T11 - TW);
+	       T12 = TW + T11;
+	       T1l = FNMS(KP500000000, T12, TR);
+	       T1F = T1m + T1n;
+	       T1G = FNMS(KP500000000, T1F, T1E);
+	  }
+	  {
+	       E Ts, T1c, Tn, T1b;
+	       {
+		    E Tf, Th, Te, Tg;
+		    Tf = ri[WS(ios, 6)];
+		    Th = ii[WS(ios, 6)];
+		    Te = W[10];
+		    Tg = W[11];
+		    Ti = FMA(Te, Tf, Tg * Th);
+		    T1S = FNMS(Tg, Tf, Te * Th);
+	       }
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = ri[WS(ios, 2)];
+		    Tr = ii[WS(ios, 2)];
+		    To = W[2];
+		    Tq = W[3];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    T1c = FNMS(Tq, Tp, To * Tr);
+	       }
+	       {
+		    E Tk, Tm, Tj, Tl;
+		    Tk = ri[WS(ios, 10)];
+		    Tm = ii[WS(ios, 10)];
+		    Tj = W[18];
+		    Tl = W[19];
+		    Tn = FMA(Tj, Tk, Tl * Tm);
+		    T1b = FNMS(Tl, Tk, Tj * Tm);
+	       }
+	       T1d = KP866025403 * (T1b - T1c);
+	       T24 = KP866025403 * (Ts - Tn);
+	       Tt = Tn + Ts;
+	       T1a = FNMS(KP500000000, Tt, Ti);
+	       T1T = T1b + T1c;
+	       T25 = FNMS(KP500000000, T1T, T1S);
+	  }
+	  {
+	       E TK, T1i, TF, T1h;
+	       {
+		    E Tx, Tz, Tw, Ty;
+		    Tx = ri[WS(ios, 3)];
+		    Tz = ii[WS(ios, 3)];
+		    Tw = W[4];
+		    Ty = W[5];
+		    TA = FMA(Tw, Tx, Ty * Tz);
+		    T1z = FNMS(Ty, Tx, Tw * Tz);
+	       }
+	       {
+		    E TH, TJ, TG, TI;
+		    TH = ri[WS(ios, 11)];
+		    TJ = ii[WS(ios, 11)];
+		    TG = W[20];
+		    TI = W[21];
+		    TK = FMA(TG, TH, TI * TJ);
+		    T1i = FNMS(TI, TH, TG * TJ);
+	       }
+	       {
+		    E TC, TE, TB, TD;
+		    TC = ri[WS(ios, 7)];
+		    TE = ii[WS(ios, 7)];
+		    TB = W[12];
+		    TD = W[13];
+		    TF = FMA(TB, TC, TD * TE);
+		    T1h = FNMS(TD, TC, TB * TE);
+	       }
+	       T1j = KP866025403 * (T1h - T1i);
+	       T1y = KP866025403 * (TK - TF);
+	       TL = TF + TK;
+	       T1g = FNMS(KP500000000, TL, TA);
+	       T1A = T1h + T1i;
+	       T1B = FNMS(KP500000000, T1A, T1z);
+	  }
+	  {
+	       E Tv, T1N, T1Y, T20, T14, T1Z, T1Q, T1R;
+	       {
+		    E Td, Tu, T1U, T1X;
+		    Td = T1 + Tc;
+		    Tu = Ti + Tt;
+		    Tv = Td + Tu;
+		    T1N = Td - Tu;
+		    T1U = T1S + T1T;
+		    T1X = T1V + T1W;
+		    T1Y = T1U + T1X;
+		    T20 = T1X - T1U;
+	       }
+	       {
+		    E TM, T13, T1O, T1P;
+		    TM = TA + TL;
+		    T13 = TR + T12;
+		    T14 = TM + T13;
+		    T1Z = TM - T13;
+		    T1O = T1z + T1A;
+		    T1P = T1E + T1F;
+		    T1Q = T1O - T1P;
+		    T1R = T1O + T1P;
+	       }
+	       ri[WS(ios, 6)] = Tv - T14;
+	       ii[WS(ios, 6)] = T1Y - T1R;
+	       ri[0] = Tv + T14;
+	       ii[0] = T1R + T1Y;
+	       ri[WS(ios, 3)] = T1N - T1Q;
+	       ii[WS(ios, 3)] = T1Z + T20;
+	       ri[WS(ios, 9)] = T1N + T1Q;
+	       ii[WS(ios, 9)] = T20 - T1Z;
+	  }
+	  {
+	       E T1t, T1x, T27, T2a, T1w, T28, T1I, T29;
+	       {
+		    E T1r, T1s, T23, T26;
+		    T1r = T15 + T18;
+		    T1s = T1a + T1d;
+		    T1t = T1r + T1s;
+		    T1x = T1r - T1s;
+		    T23 = T21 + T22;
+		    T26 = T24 + T25;
+		    T27 = T23 - T26;
+		    T2a = T26 + T23;
+	       }
+	       {
+		    E T1u, T1v, T1C, T1H;
+		    T1u = T1g + T1j;
+		    T1v = T1l + T1o;
+		    T1w = T1u + T1v;
+		    T28 = T1u - T1v;
+		    T1C = T1y + T1B;
+		    T1H = T1D + T1G;
+		    T1I = T1C - T1H;
+		    T29 = T1C + T1H;
+	       }
+	       ri[WS(ios, 10)] = T1t - T1w;
+	       ii[WS(ios, 10)] = T2a - T29;
+	       ri[WS(ios, 4)] = T1t + T1w;
+	       ii[WS(ios, 4)] = T29 + T2a;
+	       ri[WS(ios, 7)] = T1x - T1I;
+	       ii[WS(ios, 7)] = T28 + T27;
+	       ri[WS(ios, 1)] = T1x + T1I;
+	       ii[WS(ios, 1)] = T27 - T28;
+	  }
+	  {
+	       E T1f, T1J, T2d, T2f, T1q, T2g, T1M, T2e;
+	       {
+		    E T19, T1e, T2b, T2c;
+		    T19 = T15 - T18;
+		    T1e = T1a - T1d;
+		    T1f = T19 + T1e;
+		    T1J = T19 - T1e;
+		    T2b = T25 - T24;
+		    T2c = T22 - T21;
+		    T2d = T2b + T2c;
+		    T2f = T2c - T2b;
+	       }
+	       {
+		    E T1k, T1p, T1K, T1L;
+		    T1k = T1g - T1j;
+		    T1p = T1l - T1o;
+		    T1q = T1k + T1p;
+		    T2g = T1k - T1p;
+		    T1K = T1B - T1y;
+		    T1L = T1G - T1D;
+		    T1M = T1K - T1L;
+		    T2e = T1K + T1L;
+	       }
+	       ri[WS(ios, 2)] = T1f - T1q;
+	       ii[WS(ios, 2)] = T2d - T2e;
+	       ri[WS(ios, 8)] = T1f + T1q;
+	       ii[WS(ios, 8)] = T2e + T2d;
+	       ri[WS(ios, 11)] = T1J - T1M;
+	       ii[WS(ios, 11)] = T2g + T2f;
+	       ri[WS(ios, 5)] = T1J + T1M;
+	       ii[WS(ios, 5)] = T2f - T2g;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 12},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 12, "t1_12", twinstr, {88, 30, 30, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_12) (planner *p) {
+     X(kdft_dit_register) (p, t1_12, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_15.c b/src/fftw3/dft/codelets/standard/t1_15.c
new file mode 100644
index 0000000..1c721cc
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_15.c
@@ -0,0 +1,405 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:07 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 15 -name t1_15 -include t.h */
+
+/*
+ * This function contains 184 FP additions, 112 FP multiplications,
+ * (or, 128 additions, 56 multiplications, 56 fused multiply/add),
+ * 65 stack variables, and 60 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_15.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_15.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_15.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_15(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 28) {
+	  E T1q, T34, Td, T1n, T2S, T35, T13, T1k, T1l, T2E, T2F, T2O, T1H, T1T, T2k;
+	  E T2t, T2f, T2s, T1M, T1U, Tu, TL, TM, T2H, T2I, T2N, T1w, T1Q, T29, T2w;
+	  E T24, T2v, T1B, T1R;
+	  {
+	       E T1, T2R, T6, T1o, Tb, T1p, Tc, T2Q;
+	       T1 = ri[0];
+	       T2R = ii[0];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = ri[WS(ios, 5)];
+		    T5 = ii[WS(ios, 5)];
+		    T2 = W[8];
+		    T4 = W[9];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T1o = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T8, Ta, T7, T9;
+		    T8 = ri[WS(ios, 10)];
+		    Ta = ii[WS(ios, 10)];
+		    T7 = W[18];
+		    T9 = W[19];
+		    Tb = FMA(T7, T8, T9 * Ta);
+		    T1p = FNMS(T9, T8, T7 * Ta);
+	       }
+	       T1q = KP866025403 * (T1o - T1p);
+	       T34 = KP866025403 * (Tb - T6);
+	       Tc = T6 + Tb;
+	       Td = T1 + Tc;
+	       T1n = FNMS(KP500000000, Tc, T1);
+	       T2Q = T1o + T1p;
+	       T2S = T2Q + T2R;
+	       T35 = FNMS(KP500000000, T2Q, T2R);
+	  }
+	  {
+	       E TR, T2c, T18, T2h, TW, T1E, T11, T1F, T12, T2d, T1d, T1J, T1i, T1K, T1j;
+	       E T2i;
+	       {
+		    E TO, TQ, TN, TP;
+		    TO = ri[WS(ios, 6)];
+		    TQ = ii[WS(ios, 6)];
+		    TN = W[10];
+		    TP = W[11];
+		    TR = FMA(TN, TO, TP * TQ);
+		    T2c = FNMS(TP, TO, TN * TQ);
+	       }
+	       {
+		    E T15, T17, T14, T16;
+		    T15 = ri[WS(ios, 9)];
+		    T17 = ii[WS(ios, 9)];
+		    T14 = W[16];
+		    T16 = W[17];
+		    T18 = FMA(T14, T15, T16 * T17);
+		    T2h = FNMS(T16, T15, T14 * T17);
+	       }
+	       {
+		    E TT, TV, TS, TU;
+		    TT = ri[WS(ios, 11)];
+		    TV = ii[WS(ios, 11)];
+		    TS = W[20];
+		    TU = W[21];
+		    TW = FMA(TS, TT, TU * TV);
+		    T1E = FNMS(TU, TT, TS * TV);
+	       }
+	       {
+		    E TY, T10, TX, TZ;
+		    TY = ri[WS(ios, 1)];
+		    T10 = ii[WS(ios, 1)];
+		    TX = W[0];
+		    TZ = W[1];
+		    T11 = FMA(TX, TY, TZ * T10);
+		    T1F = FNMS(TZ, TY, TX * T10);
+	       }
+	       T12 = TW + T11;
+	       T2d = T1E + T1F;
+	       {
+		    E T1a, T1c, T19, T1b;
+		    T1a = ri[WS(ios, 14)];
+		    T1c = ii[WS(ios, 14)];
+		    T19 = W[26];
+		    T1b = W[27];
+		    T1d = FMA(T19, T1a, T1b * T1c);
+		    T1J = FNMS(T1b, T1a, T19 * T1c);
+	       }
+	       {
+		    E T1f, T1h, T1e, T1g;
+		    T1f = ri[WS(ios, 4)];
+		    T1h = ii[WS(ios, 4)];
+		    T1e = W[6];
+		    T1g = W[7];
+		    T1i = FMA(T1e, T1f, T1g * T1h);
+		    T1K = FNMS(T1g, T1f, T1e * T1h);
+	       }
+	       T1j = T1d + T1i;
+	       T2i = T1J + T1K;
+	       {
+		    E T1D, T1G, T2g, T2j;
+		    T13 = TR + T12;
+		    T1k = T18 + T1j;
+		    T1l = T13 + T1k;
+		    T2E = T2c + T2d;
+		    T2F = T2h + T2i;
+		    T2O = T2E + T2F;
+		    T1D = FNMS(KP500000000, T12, TR);
+		    T1G = KP866025403 * (T1E - T1F);
+		    T1H = T1D - T1G;
+		    T1T = T1D + T1G;
+		    T2g = KP866025403 * (T1i - T1d);
+		    T2j = FNMS(KP500000000, T2i, T2h);
+		    T2k = T2g + T2j;
+		    T2t = T2j - T2g;
+		    {
+			 E T2b, T2e, T1I, T1L;
+			 T2b = KP866025403 * (T11 - TW);
+			 T2e = FNMS(KP500000000, T2d, T2c);
+			 T2f = T2b + T2e;
+			 T2s = T2e - T2b;
+			 T1I = FNMS(KP500000000, T1j, T18);
+			 T1L = KP866025403 * (T1J - T1K);
+			 T1M = T1I - T1L;
+			 T1U = T1I + T1L;
+		    }
+	       }
+	  }
+	  {
+	       E Ti, T21, Tz, T26, Tn, T1t, Ts, T1u, Tt, T22, TE, T1y, TJ, T1z, TK;
+	       E T27;
+	       {
+		    E Tf, Th, Te, Tg;
+		    Tf = ri[WS(ios, 3)];
+		    Th = ii[WS(ios, 3)];
+		    Te = W[4];
+		    Tg = W[5];
+		    Ti = FMA(Te, Tf, Tg * Th);
+		    T21 = FNMS(Tg, Tf, Te * Th);
+	       }
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = ri[WS(ios, 12)];
+		    Ty = ii[WS(ios, 12)];
+		    Tv = W[22];
+		    Tx = W[23];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T26 = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E Tk, Tm, Tj, Tl;
+		    Tk = ri[WS(ios, 8)];
+		    Tm = ii[WS(ios, 8)];
+		    Tj = W[14];
+		    Tl = W[15];
+		    Tn = FMA(Tj, Tk, Tl * Tm);
+		    T1t = FNMS(Tl, Tk, Tj * Tm);
+	       }
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = ri[WS(ios, 13)];
+		    Tr = ii[WS(ios, 13)];
+		    To = W[24];
+		    Tq = W[25];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    T1u = FNMS(Tq, Tp, To * Tr);
+	       }
+	       Tt = Tn + Ts;
+	       T22 = T1t + T1u;
+	       {
+		    E TB, TD, TA, TC;
+		    TB = ri[WS(ios, 2)];
+		    TD = ii[WS(ios, 2)];
+		    TA = W[2];
+		    TC = W[3];
+		    TE = FMA(TA, TB, TC * TD);
+		    T1y = FNMS(TC, TB, TA * TD);
+	       }
+	       {
+		    E TG, TI, TF, TH;
+		    TG = ri[WS(ios, 7)];
+		    TI = ii[WS(ios, 7)];
+		    TF = W[12];
+		    TH = W[13];
+		    TJ = FMA(TF, TG, TH * TI);
+		    T1z = FNMS(TH, TG, TF * TI);
+	       }
+	       TK = TE + TJ;
+	       T27 = T1y + T1z;
+	       {
+		    E T1s, T1v, T25, T28;
+		    Tu = Ti + Tt;
+		    TL = Tz + TK;
+		    TM = Tu + TL;
+		    T2H = T21 + T22;
+		    T2I = T26 + T27;
+		    T2N = T2H + T2I;
+		    T1s = FNMS(KP500000000, Tt, Ti);
+		    T1v = KP866025403 * (T1t - T1u);
+		    T1w = T1s - T1v;
+		    T1Q = T1s + T1v;
+		    T25 = KP866025403 * (TJ - TE);
+		    T28 = FNMS(KP500000000, T27, T26);
+		    T29 = T25 + T28;
+		    T2w = T28 - T25;
+		    {
+			 E T20, T23, T1x, T1A;
+			 T20 = KP866025403 * (Ts - Tn);
+			 T23 = FNMS(KP500000000, T22, T21);
+			 T24 = T20 + T23;
+			 T2v = T23 - T20;
+			 T1x = FNMS(KP500000000, TK, Tz);
+			 T1A = KP866025403 * (T1y - T1z);
+			 T1B = T1x - T1A;
+			 T1R = T1x + T1A;
+		    }
+	       }
+	  }
+	  {
+	       E T2C, T1m, T2B, T2K, T2M, T2G, T2J, T2L, T2D;
+	       T2C = KP559016994 * (TM - T1l);
+	       T1m = TM + T1l;
+	       T2B = FNMS(KP250000000, T1m, Td);
+	       T2G = T2E - T2F;
+	       T2J = T2H - T2I;
+	       T2K = FNMS(KP587785252, T2J, KP951056516 * T2G);
+	       T2M = FMA(KP951056516, T2J, KP587785252 * T2G);
+	       ri[0] = Td + T1m;
+	       T2L = T2C + T2B;
+	       ri[WS(ios, 9)] = T2L - T2M;
+	       ri[WS(ios, 6)] = T2L + T2M;
+	       T2D = T2B - T2C;
+	       ri[WS(ios, 12)] = T2D - T2K;
+	       ri[WS(ios, 3)] = T2D + T2K;
+	  }
+	  {
+	       E T2U, T2P, T2T, T2Y, T30, T2W, T2X, T2Z, T2V;
+	       T2U = KP559016994 * (T2N - T2O);
+	       T2P = T2N + T2O;
+	       T2T = FNMS(KP250000000, T2P, T2S);
+	       T2W = T13 - T1k;
+	       T2X = Tu - TL;
+	       T2Y = FNMS(KP587785252, T2X, KP951056516 * T2W);
+	       T30 = FMA(KP951056516, T2X, KP587785252 * T2W);
+	       ii[0] = T2P + T2S;
+	       T2Z = T2U + T2T;
+	       ii[WS(ios, 6)] = T2Z - T30;
+	       ii[WS(ios, 9)] = T30 + T2Z;
+	       T2V = T2T - T2U;
+	       ii[WS(ios, 3)] = T2V - T2Y;
+	       ii[WS(ios, 12)] = T2Y + T2V;
+	  }
+	  {
+	       E T2y, T2A, T1r, T1O, T2p, T2q, T2z, T2r;
+	       {
+		    E T2u, T2x, T1C, T1N;
+		    T2u = T2s - T2t;
+		    T2x = T2v - T2w;
+		    T2y = FNMS(KP587785252, T2x, KP951056516 * T2u);
+		    T2A = FMA(KP951056516, T2x, KP587785252 * T2u);
+		    T1r = T1n - T1q;
+		    T1C = T1w + T1B;
+		    T1N = T1H + T1M;
+		    T1O = T1C + T1N;
+		    T2p = FNMS(KP250000000, T1O, T1r);
+		    T2q = KP559016994 * (T1C - T1N);
+	       }
+	       ri[WS(ios, 5)] = T1r + T1O;
+	       T2z = T2q + T2p;
+	       ri[WS(ios, 14)] = T2z - T2A;
+	       ri[WS(ios, 11)] = T2z + T2A;
+	       T2r = T2p - T2q;
+	       ri[WS(ios, 2)] = T2r - T2y;
+	       ri[WS(ios, 8)] = T2r + T2y;
+	  }
+	  {
+	       E T3h, T3q, T3i, T3l, T3m, T3n, T3p, T3o;
+	       {
+		    E T3f, T3g, T3j, T3k;
+		    T3f = T1H - T1M;
+		    T3g = T1w - T1B;
+		    T3h = FNMS(KP587785252, T3g, KP951056516 * T3f);
+		    T3q = FMA(KP951056516, T3g, KP587785252 * T3f);
+		    T3i = T35 - T34;
+		    T3j = T2v + T2w;
+		    T3k = T2s + T2t;
+		    T3l = T3j + T3k;
+		    T3m = FNMS(KP250000000, T3l, T3i);
+		    T3n = KP559016994 * (T3j - T3k);
+	       }
+	       ii[WS(ios, 5)] = T3l + T3i;
+	       T3p = T3n + T3m;
+	       ii[WS(ios, 11)] = T3p - T3q;
+	       ii[WS(ios, 14)] = T3q + T3p;
+	       T3o = T3m - T3n;
+	       ii[WS(ios, 2)] = T3h + T3o;
+	       ii[WS(ios, 8)] = T3o - T3h;
+	  }
+	  {
+	       E T3c, T3d, T36, T37, T33, T38, T3e, T39;
+	       {
+		    E T3a, T3b, T31, T32;
+		    T3a = T1Q - T1R;
+		    T3b = T1T - T1U;
+		    T3c = FMA(KP951056516, T3a, KP587785252 * T3b);
+		    T3d = FNMS(KP587785252, T3a, KP951056516 * T3b);
+		    T36 = T34 + T35;
+		    T31 = T24 + T29;
+		    T32 = T2f + T2k;
+		    T37 = T31 + T32;
+		    T33 = KP559016994 * (T31 - T32);
+		    T38 = FNMS(KP250000000, T37, T36);
+	       }
+	       ii[WS(ios, 10)] = T37 + T36;
+	       T3e = T38 - T33;
+	       ii[WS(ios, 7)] = T3d + T3e;
+	       ii[WS(ios, 13)] = T3e - T3d;
+	       T39 = T33 + T38;
+	       ii[WS(ios, 1)] = T39 - T3c;
+	       ii[WS(ios, 4)] = T3c + T39;
+	  }
+	  {
+	       E T2m, T2o, T1P, T1W, T1X, T1Y, T2n, T1Z;
+	       {
+		    E T2a, T2l, T1S, T1V;
+		    T2a = T24 - T29;
+		    T2l = T2f - T2k;
+		    T2m = FMA(KP951056516, T2a, KP587785252 * T2l);
+		    T2o = FNMS(KP587785252, T2a, KP951056516 * T2l);
+		    T1P = T1n + T1q;
+		    T1S = T1Q + T1R;
+		    T1V = T1T + T1U;
+		    T1W = T1S + T1V;
+		    T1X = KP559016994 * (T1S - T1V);
+		    T1Y = FNMS(KP250000000, T1W, T1P);
+	       }
+	       ri[WS(ios, 10)] = T1P + T1W;
+	       T2n = T1Y - T1X;
+	       ri[WS(ios, 7)] = T2n - T2o;
+	       ri[WS(ios, 13)] = T2n + T2o;
+	       T1Z = T1X + T1Y;
+	       ri[WS(ios, 4)] = T1Z - T2m;
+	       ri[WS(ios, 1)] = T1Z + T2m;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 15},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 15, "t1_15", twinstr, {128, 56, 56, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_15) (planner *p) {
+     X(kdft_dit_register) (p, t1_15, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_16.c b/src/fftw3/dft/codelets/standard/t1_16.c
new file mode 100644
index 0000000..6dc13f9
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_16.c
@@ -0,0 +1,406 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:07 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 16 -name t1_16 -include t.h */
+
+/*
+ * This function contains 174 FP additions, 84 FP multiplications,
+ * (or, 136 additions, 46 multiplications, 38 fused multiply/add),
+ * 52 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_16(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 30) {
+	  E T7, T37, T1t, T2U, Ti, T38, T1w, T2R, Tu, T2s, T1C, T2c, TF, T2t, T1H;
+	  E T2d, T1f, T1q, T2B, T2C, T2D, T2E, T1Z, T2j, T24, T2k, TS, T13, T2w, T2x;
+	  E T2y, T2z, T1O, T2g, T1T, T2h;
+	  {
+	       E T1, T2T, T6, T2S;
+	       T1 = ri[0];
+	       T2T = ii[0];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = ri[WS(ios, 8)];
+		    T5 = ii[WS(ios, 8)];
+		    T2 = W[14];
+		    T4 = W[15];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T2S = FNMS(T4, T3, T2 * T5);
+	       }
+	       T7 = T1 + T6;
+	       T37 = T2T - T2S;
+	       T1t = T1 - T6;
+	       T2U = T2S + T2T;
+	  }
+	  {
+	       E Tc, T1u, Th, T1v;
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = ri[WS(ios, 4)];
+		    Tb = ii[WS(ios, 4)];
+		    T8 = W[6];
+		    Ta = W[7];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    T1u = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = ri[WS(ios, 12)];
+		    Tg = ii[WS(ios, 12)];
+		    Td = W[22];
+		    Tf = W[23];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    T1v = FNMS(Tf, Te, Td * Tg);
+	       }
+	       Ti = Tc + Th;
+	       T38 = Tc - Th;
+	       T1w = T1u - T1v;
+	       T2R = T1u + T1v;
+	  }
+	  {
+	       E To, T1y, Tt, T1z, T1A, T1B;
+	       {
+		    E Tl, Tn, Tk, Tm;
+		    Tl = ri[WS(ios, 2)];
+		    Tn = ii[WS(ios, 2)];
+		    Tk = W[2];
+		    Tm = W[3];
+		    To = FMA(Tk, Tl, Tm * Tn);
+		    T1y = FNMS(Tm, Tl, Tk * Tn);
+	       }
+	       {
+		    E Tq, Ts, Tp, Tr;
+		    Tq = ri[WS(ios, 10)];
+		    Ts = ii[WS(ios, 10)];
+		    Tp = W[18];
+		    Tr = W[19];
+		    Tt = FMA(Tp, Tq, Tr * Ts);
+		    T1z = FNMS(Tr, Tq, Tp * Ts);
+	       }
+	       Tu = To + Tt;
+	       T2s = T1y + T1z;
+	       T1A = T1y - T1z;
+	       T1B = To - Tt;
+	       T1C = T1A - T1B;
+	       T2c = T1B + T1A;
+	  }
+	  {
+	       E Tz, T1E, TE, T1F, T1D, T1G;
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = ri[WS(ios, 14)];
+		    Ty = ii[WS(ios, 14)];
+		    Tv = W[26];
+		    Tx = W[27];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T1E = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = ri[WS(ios, 6)];
+		    TD = ii[WS(ios, 6)];
+		    TA = W[10];
+		    TC = W[11];
+		    TE = FMA(TA, TB, TC * TD);
+		    T1F = FNMS(TC, TB, TA * TD);
+	       }
+	       TF = Tz + TE;
+	       T2t = T1E + T1F;
+	       T1D = Tz - TE;
+	       T1G = T1E - T1F;
+	       T1H = T1D + T1G;
+	       T2d = T1D - T1G;
+	  }
+	  {
+	       E T19, T20, T1p, T1X, T1e, T21, T1k, T1W;
+	       {
+		    E T16, T18, T15, T17;
+		    T16 = ri[WS(ios, 15)];
+		    T18 = ii[WS(ios, 15)];
+		    T15 = W[28];
+		    T17 = W[29];
+		    T19 = FMA(T15, T16, T17 * T18);
+		    T20 = FNMS(T17, T16, T15 * T18);
+	       }
+	       {
+		    E T1m, T1o, T1l, T1n;
+		    T1m = ri[WS(ios, 11)];
+		    T1o = ii[WS(ios, 11)];
+		    T1l = W[20];
+		    T1n = W[21];
+		    T1p = FMA(T1l, T1m, T1n * T1o);
+		    T1X = FNMS(T1n, T1m, T1l * T1o);
+	       }
+	       {
+		    E T1b, T1d, T1a, T1c;
+		    T1b = ri[WS(ios, 7)];
+		    T1d = ii[WS(ios, 7)];
+		    T1a = W[12];
+		    T1c = W[13];
+		    T1e = FMA(T1a, T1b, T1c * T1d);
+		    T21 = FNMS(T1c, T1b, T1a * T1d);
+	       }
+	       {
+		    E T1h, T1j, T1g, T1i;
+		    T1h = ri[WS(ios, 3)];
+		    T1j = ii[WS(ios, 3)];
+		    T1g = W[4];
+		    T1i = W[5];
+		    T1k = FMA(T1g, T1h, T1i * T1j);
+		    T1W = FNMS(T1i, T1h, T1g * T1j);
+	       }
+	       T1f = T19 + T1e;
+	       T1q = T1k + T1p;
+	       T2B = T1f - T1q;
+	       T2C = T20 + T21;
+	       T2D = T1W + T1X;
+	       T2E = T2C - T2D;
+	       {
+		    E T1V, T1Y, T22, T23;
+		    T1V = T19 - T1e;
+		    T1Y = T1W - T1X;
+		    T1Z = T1V - T1Y;
+		    T2j = T1V + T1Y;
+		    T22 = T20 - T21;
+		    T23 = T1k - T1p;
+		    T24 = T22 + T23;
+		    T2k = T22 - T23;
+	       }
+	  }
+	  {
+	       E TM, T1K, T12, T1R, TR, T1L, TX, T1Q;
+	       {
+		    E TJ, TL, TI, TK;
+		    TJ = ri[WS(ios, 1)];
+		    TL = ii[WS(ios, 1)];
+		    TI = W[0];
+		    TK = W[1];
+		    TM = FMA(TI, TJ, TK * TL);
+		    T1K = FNMS(TK, TJ, TI * TL);
+	       }
+	       {
+		    E TZ, T11, TY, T10;
+		    TZ = ri[WS(ios, 13)];
+		    T11 = ii[WS(ios, 13)];
+		    TY = W[24];
+		    T10 = W[25];
+		    T12 = FMA(TY, TZ, T10 * T11);
+		    T1R = FNMS(T10, TZ, TY * T11);
+	       }
+	       {
+		    E TO, TQ, TN, TP;
+		    TO = ri[WS(ios, 9)];
+		    TQ = ii[WS(ios, 9)];
+		    TN = W[16];
+		    TP = W[17];
+		    TR = FMA(TN, TO, TP * TQ);
+		    T1L = FNMS(TP, TO, TN * TQ);
+	       }
+	       {
+		    E TU, TW, TT, TV;
+		    TU = ri[WS(ios, 5)];
+		    TW = ii[WS(ios, 5)];
+		    TT = W[8];
+		    TV = W[9];
+		    TX = FMA(TT, TU, TV * TW);
+		    T1Q = FNMS(TV, TU, TT * TW);
+	       }
+	       TS = TM + TR;
+	       T13 = TX + T12;
+	       T2w = TS - T13;
+	       T2x = T1K + T1L;
+	       T2y = T1Q + T1R;
+	       T2z = T2x - T2y;
+	       {
+		    E T1M, T1N, T1P, T1S;
+		    T1M = T1K - T1L;
+		    T1N = TX - T12;
+		    T1O = T1M + T1N;
+		    T2g = T1M - T1N;
+		    T1P = TM - TR;
+		    T1S = T1Q - T1R;
+		    T1T = T1P - T1S;
+		    T2h = T1P + T1S;
+	       }
+	  }
+	  {
+	       E T1J, T27, T3g, T3i, T26, T3h, T2a, T3d;
+	       {
+		    E T1x, T1I, T3e, T3f;
+		    T1x = T1t - T1w;
+		    T1I = KP707106781 * (T1C - T1H);
+		    T1J = T1x + T1I;
+		    T27 = T1x - T1I;
+		    T3e = KP707106781 * (T2d - T2c);
+		    T3f = T38 + T37;
+		    T3g = T3e + T3f;
+		    T3i = T3f - T3e;
+	       }
+	       {
+		    E T1U, T25, T28, T29;
+		    T1U = FMA(KP923879532, T1O, KP382683432 * T1T);
+		    T25 = FNMS(KP923879532, T24, KP382683432 * T1Z);
+		    T26 = T1U + T25;
+		    T3h = T25 - T1U;
+		    T28 = FNMS(KP923879532, T1T, KP382683432 * T1O);
+		    T29 = FMA(KP382683432, T24, KP923879532 * T1Z);
+		    T2a = T28 - T29;
+		    T3d = T28 + T29;
+	       }
+	       ri[WS(ios, 11)] = T1J - T26;
+	       ii[WS(ios, 11)] = T3g - T3d;
+	       ri[WS(ios, 3)] = T1J + T26;
+	       ii[WS(ios, 3)] = T3d + T3g;
+	       ri[WS(ios, 15)] = T27 - T2a;
+	       ii[WS(ios, 15)] = T3i - T3h;
+	       ri[WS(ios, 7)] = T27 + T2a;
+	       ii[WS(ios, 7)] = T3h + T3i;
+	  }
+	  {
+	       E T2v, T2H, T32, T34, T2G, T33, T2K, T2Z;
+	       {
+		    E T2r, T2u, T30, T31;
+		    T2r = T7 - Ti;
+		    T2u = T2s - T2t;
+		    T2v = T2r + T2u;
+		    T2H = T2r - T2u;
+		    T30 = TF - Tu;
+		    T31 = T2U - T2R;
+		    T32 = T30 + T31;
+		    T34 = T31 - T30;
+	       }
+	       {
+		    E T2A, T2F, T2I, T2J;
+		    T2A = T2w + T2z;
+		    T2F = T2B - T2E;
+		    T2G = KP707106781 * (T2A + T2F);
+		    T33 = KP707106781 * (T2F - T2A);
+		    T2I = T2z - T2w;
+		    T2J = T2B + T2E;
+		    T2K = KP707106781 * (T2I - T2J);
+		    T2Z = KP707106781 * (T2I + T2J);
+	       }
+	       ri[WS(ios, 10)] = T2v - T2G;
+	       ii[WS(ios, 10)] = T32 - T2Z;
+	       ri[WS(ios, 2)] = T2v + T2G;
+	       ii[WS(ios, 2)] = T2Z + T32;
+	       ri[WS(ios, 14)] = T2H - T2K;
+	       ii[WS(ios, 14)] = T34 - T33;
+	       ri[WS(ios, 6)] = T2H + T2K;
+	       ii[WS(ios, 6)] = T33 + T34;
+	  }
+	  {
+	       E T2f, T2n, T3a, T3c, T2m, T3b, T2q, T35;
+	       {
+		    E T2b, T2e, T36, T39;
+		    T2b = T1t + T1w;
+		    T2e = KP707106781 * (T2c + T2d);
+		    T2f = T2b + T2e;
+		    T2n = T2b - T2e;
+		    T36 = KP707106781 * (T1C + T1H);
+		    T39 = T37 - T38;
+		    T3a = T36 + T39;
+		    T3c = T39 - T36;
+	       }
+	       {
+		    E T2i, T2l, T2o, T2p;
+		    T2i = FMA(KP382683432, T2g, KP923879532 * T2h);
+		    T2l = FNMS(KP382683432, T2k, KP923879532 * T2j);
+		    T2m = T2i + T2l;
+		    T3b = T2l - T2i;
+		    T2o = FNMS(KP382683432, T2h, KP923879532 * T2g);
+		    T2p = FMA(KP923879532, T2k, KP382683432 * T2j);
+		    T2q = T2o - T2p;
+		    T35 = T2o + T2p;
+	       }
+	       ri[WS(ios, 9)] = T2f - T2m;
+	       ii[WS(ios, 9)] = T3a - T35;
+	       ri[WS(ios, 1)] = T2f + T2m;
+	       ii[WS(ios, 1)] = T35 + T3a;
+	       ri[WS(ios, 13)] = T2n - T2q;
+	       ii[WS(ios, 13)] = T3c - T3b;
+	       ri[WS(ios, 5)] = T2n + T2q;
+	       ii[WS(ios, 5)] = T3b + T3c;
+	  }
+	  {
+	       E TH, T2L, T2W, T2Y, T1s, T2X, T2O, T2P;
+	       {
+		    E Tj, TG, T2Q, T2V;
+		    Tj = T7 + Ti;
+		    TG = Tu + TF;
+		    TH = Tj + TG;
+		    T2L = Tj - TG;
+		    T2Q = T2s + T2t;
+		    T2V = T2R + T2U;
+		    T2W = T2Q + T2V;
+		    T2Y = T2V - T2Q;
+	       }
+	       {
+		    E T14, T1r, T2M, T2N;
+		    T14 = TS + T13;
+		    T1r = T1f + T1q;
+		    T1s = T14 + T1r;
+		    T2X = T1r - T14;
+		    T2M = T2x + T2y;
+		    T2N = T2C + T2D;
+		    T2O = T2M - T2N;
+		    T2P = T2M + T2N;
+	       }
+	       ri[WS(ios, 8)] = TH - T1s;
+	       ii[WS(ios, 8)] = T2W - T2P;
+	       ri[0] = TH + T1s;
+	       ii[0] = T2P + T2W;
+	       ri[WS(ios, 12)] = T2L - T2O;
+	       ii[WS(ios, 12)] = T2Y - T2X;
+	       ri[WS(ios, 4)] = T2L + T2O;
+	       ii[WS(ios, 4)] = T2X + T2Y;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 16},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 16, "t1_16", twinstr, {136, 46, 38, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_16) (planner *p) {
+     X(kdft_dit_register) (p, t1_16, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_2.c b/src/fftw3/dft/codelets/standard/t1_2.c
new file mode 100644
index 0000000..9213d04
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_2.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:51 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 2 -name t1_2 -include t.h */
+
+/*
+ * This function contains 6 FP additions, 4 FP multiplications,
+ * (or, 4 additions, 2 multiplications, 2 fused multiply/add),
+ * 9 stack variables, and 8 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_2.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_2.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_2.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_2(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 2) {
+	  E T1, T8, T6, T7;
+	  T1 = ri[0];
+	  T8 = ii[0];
+	  {
+	       E T3, T5, T2, T4;
+	       T3 = ri[WS(ios, 1)];
+	       T5 = ii[WS(ios, 1)];
+	       T2 = W[0];
+	       T4 = W[1];
+	       T6 = FMA(T2, T3, T4 * T5);
+	       T7 = FNMS(T4, T3, T2 * T5);
+	  }
+	  ri[WS(ios, 1)] = T1 - T6;
+	  ii[WS(ios, 1)] = T8 - T7;
+	  ri[0] = T1 + T6;
+	  ii[0] = T7 + T8;
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 2},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 2, "t1_2", twinstr, {4, 2, 2, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_2) (planner *p) {
+     X(kdft_dit_register) (p, t1_2, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_3.c b/src/fftw3/dft/codelets/standard/t1_3.c
new file mode 100644
index 0000000..e8cfbc6
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_3.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:51 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 3 -name t1_3 -include t.h */
+
+/*
+ * This function contains 16 FP additions, 12 FP multiplications,
+ * (or, 10 additions, 6 multiplications, 6 fused multiply/add),
+ * 15 stack variables, and 12 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_3.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_3.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_3.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_3(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 4) {
+	  E T1, Ti, T6, Te, Tb, Tf, Tc, Th;
+	  T1 = ri[0];
+	  Ti = ii[0];
+	  {
+	       E T3, T5, T2, T4;
+	       T3 = ri[WS(ios, 1)];
+	       T5 = ii[WS(ios, 1)];
+	       T2 = W[0];
+	       T4 = W[1];
+	       T6 = FMA(T2, T3, T4 * T5);
+	       Te = FNMS(T4, T3, T2 * T5);
+	  }
+	  {
+	       E T8, Ta, T7, T9;
+	       T8 = ri[WS(ios, 2)];
+	       Ta = ii[WS(ios, 2)];
+	       T7 = W[2];
+	       T9 = W[3];
+	       Tb = FMA(T7, T8, T9 * Ta);
+	       Tf = FNMS(T9, T8, T7 * Ta);
+	  }
+	  Tc = T6 + Tb;
+	  Th = Te + Tf;
+	  ri[0] = T1 + Tc;
+	  ii[0] = Th + Ti;
+	  {
+	       E Td, Tg, Tj, Tk;
+	       Td = FNMS(KP500000000, Tc, T1);
+	       Tg = KP866025403 * (Te - Tf);
+	       ri[WS(ios, 2)] = Td - Tg;
+	       ri[WS(ios, 1)] = Td + Tg;
+	       Tj = KP866025403 * (Tb - T6);
+	       Tk = FNMS(KP500000000, Th, Ti);
+	       ii[WS(ios, 1)] = Tj + Tk;
+	       ii[WS(ios, 2)] = Tk - Tj;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 3},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 3, "t1_3", twinstr, {10, 6, 6, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_3) (planner *p) {
+     X(kdft_dit_register) (p, t1_3, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_32.c b/src/fftw3/dft/codelets/standard/t1_32.c
new file mode 100644
index 0000000..387b955
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_32.c
@@ -0,0 +1,892 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:07 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 32 -name t1_32 -include t.h */
+
+/*
+ * This function contains 434 FP additions, 208 FP multiplications,
+ * (or, 340 additions, 114 multiplications, 94 fused multiply/add),
+ * 96 stack variables, and 128 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_32.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_32.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_32.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_32(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 62) {
+	  E Tj, T5F, T7C, T7Q, T35, T4T, T78, T7m, T1Q, T61, T5Y, T6J, T3K, T59, T41;
+	  E T56, T2B, T67, T6e, T6O, T4b, T5d, T4s, T5g, TG, T7l, T5I, T73, T3a, T4U;
+	  E T3f, T4V, T14, T5N, T5M, T6E, T3m, T4Y, T3r, T4Z, T1r, T5P, T5S, T6F, T3x;
+	  E T51, T3C, T52, T2d, T5Z, T64, T6K, T3V, T57, T44, T5a, T2Y, T6f, T6a, T6P;
+	  E T4m, T5h, T4v, T5e;
+	  {
+	       E T1, T76, T6, T75, Tc, T32, Th, T33;
+	       T1 = ri[0];
+	       T76 = ii[0];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = ri[WS(ios, 16)];
+		    T5 = ii[WS(ios, 16)];
+		    T2 = W[30];
+		    T4 = W[31];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T75 = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = ri[WS(ios, 8)];
+		    Tb = ii[WS(ios, 8)];
+		    T8 = W[14];
+		    Ta = W[15];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    T32 = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = ri[WS(ios, 24)];
+		    Tg = ii[WS(ios, 24)];
+		    Td = W[46];
+		    Tf = W[47];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    T33 = FNMS(Tf, Te, Td * Tg);
+	       }
+	       {
+		    E T7, Ti, T7A, T7B;
+		    T7 = T1 + T6;
+		    Ti = Tc + Th;
+		    Tj = T7 + Ti;
+		    T5F = T7 - Ti;
+		    T7A = T76 - T75;
+		    T7B = Tc - Th;
+		    T7C = T7A - T7B;
+		    T7Q = T7B + T7A;
+	       }
+	       {
+		    E T31, T34, T74, T77;
+		    T31 = T1 - T6;
+		    T34 = T32 - T33;
+		    T35 = T31 - T34;
+		    T4T = T31 + T34;
+		    T74 = T32 + T33;
+		    T77 = T75 + T76;
+		    T78 = T74 + T77;
+		    T7m = T77 - T74;
+	       }
+	  }
+	  {
+	       E T1y, T3G, T1O, T3Z, T1D, T3H, T1J, T3Y;
+	       {
+		    E T1v, T1x, T1u, T1w;
+		    T1v = ri[WS(ios, 1)];
+		    T1x = ii[WS(ios, 1)];
+		    T1u = W[0];
+		    T1w = W[1];
+		    T1y = FMA(T1u, T1v, T1w * T1x);
+		    T3G = FNMS(T1w, T1v, T1u * T1x);
+	       }
+	       {
+		    E T1L, T1N, T1K, T1M;
+		    T1L = ri[WS(ios, 25)];
+		    T1N = ii[WS(ios, 25)];
+		    T1K = W[48];
+		    T1M = W[49];
+		    T1O = FMA(T1K, T1L, T1M * T1N);
+		    T3Z = FNMS(T1M, T1L, T1K * T1N);
+	       }
+	       {
+		    E T1A, T1C, T1z, T1B;
+		    T1A = ri[WS(ios, 17)];
+		    T1C = ii[WS(ios, 17)];
+		    T1z = W[32];
+		    T1B = W[33];
+		    T1D = FMA(T1z, T1A, T1B * T1C);
+		    T3H = FNMS(T1B, T1A, T1z * T1C);
+	       }
+	       {
+		    E T1G, T1I, T1F, T1H;
+		    T1G = ri[WS(ios, 9)];
+		    T1I = ii[WS(ios, 9)];
+		    T1F = W[16];
+		    T1H = W[17];
+		    T1J = FMA(T1F, T1G, T1H * T1I);
+		    T3Y = FNMS(T1H, T1G, T1F * T1I);
+	       }
+	       {
+		    E T1E, T1P, T5W, T5X;
+		    T1E = T1y + T1D;
+		    T1P = T1J + T1O;
+		    T1Q = T1E + T1P;
+		    T61 = T1E - T1P;
+		    T5W = T3G + T3H;
+		    T5X = T3Y + T3Z;
+		    T5Y = T5W - T5X;
+		    T6J = T5W + T5X;
+	       }
+	       {
+		    E T3I, T3J, T3X, T40;
+		    T3I = T3G - T3H;
+		    T3J = T1J - T1O;
+		    T3K = T3I + T3J;
+		    T59 = T3I - T3J;
+		    T3X = T1y - T1D;
+		    T40 = T3Y - T3Z;
+		    T41 = T3X - T40;
+		    T56 = T3X + T40;
+	       }
+	  }
+	  {
+	       E T2j, T4o, T2z, T49, T2o, T4p, T2u, T48;
+	       {
+		    E T2g, T2i, T2f, T2h;
+		    T2g = ri[WS(ios, 31)];
+		    T2i = ii[WS(ios, 31)];
+		    T2f = W[60];
+		    T2h = W[61];
+		    T2j = FMA(T2f, T2g, T2h * T2i);
+		    T4o = FNMS(T2h, T2g, T2f * T2i);
+	       }
+	       {
+		    E T2w, T2y, T2v, T2x;
+		    T2w = ri[WS(ios, 23)];
+		    T2y = ii[WS(ios, 23)];
+		    T2v = W[44];
+		    T2x = W[45];
+		    T2z = FMA(T2v, T2w, T2x * T2y);
+		    T49 = FNMS(T2x, T2w, T2v * T2y);
+	       }
+	       {
+		    E T2l, T2n, T2k, T2m;
+		    T2l = ri[WS(ios, 15)];
+		    T2n = ii[WS(ios, 15)];
+		    T2k = W[28];
+		    T2m = W[29];
+		    T2o = FMA(T2k, T2l, T2m * T2n);
+		    T4p = FNMS(T2m, T2l, T2k * T2n);
+	       }
+	       {
+		    E T2r, T2t, T2q, T2s;
+		    T2r = ri[WS(ios, 7)];
+		    T2t = ii[WS(ios, 7)];
+		    T2q = W[12];
+		    T2s = W[13];
+		    T2u = FMA(T2q, T2r, T2s * T2t);
+		    T48 = FNMS(T2s, T2r, T2q * T2t);
+	       }
+	       {
+		    E T2p, T2A, T6c, T6d;
+		    T2p = T2j + T2o;
+		    T2A = T2u + T2z;
+		    T2B = T2p + T2A;
+		    T67 = T2p - T2A;
+		    T6c = T4o + T4p;
+		    T6d = T48 + T49;
+		    T6e = T6c - T6d;
+		    T6O = T6c + T6d;
+	       }
+	       {
+		    E T47, T4a, T4q, T4r;
+		    T47 = T2j - T2o;
+		    T4a = T48 - T49;
+		    T4b = T47 - T4a;
+		    T5d = T47 + T4a;
+		    T4q = T4o - T4p;
+		    T4r = T2u - T2z;
+		    T4s = T4q + T4r;
+		    T5g = T4q - T4r;
+	       }
+	  }
+	  {
+	       E To, T36, TE, T3d, Tt, T37, Tz, T3c;
+	       {
+		    E Tl, Tn, Tk, Tm;
+		    Tl = ri[WS(ios, 4)];
+		    Tn = ii[WS(ios, 4)];
+		    Tk = W[6];
+		    Tm = W[7];
+		    To = FMA(Tk, Tl, Tm * Tn);
+		    T36 = FNMS(Tm, Tl, Tk * Tn);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = ri[WS(ios, 12)];
+		    TD = ii[WS(ios, 12)];
+		    TA = W[22];
+		    TC = W[23];
+		    TE = FMA(TA, TB, TC * TD);
+		    T3d = FNMS(TC, TB, TA * TD);
+	       }
+	       {
+		    E Tq, Ts, Tp, Tr;
+		    Tq = ri[WS(ios, 20)];
+		    Ts = ii[WS(ios, 20)];
+		    Tp = W[38];
+		    Tr = W[39];
+		    Tt = FMA(Tp, Tq, Tr * Ts);
+		    T37 = FNMS(Tr, Tq, Tp * Ts);
+	       }
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = ri[WS(ios, 28)];
+		    Ty = ii[WS(ios, 28)];
+		    Tv = W[54];
+		    Tx = W[55];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T3c = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E Tu, TF, T5G, T5H;
+		    Tu = To + Tt;
+		    TF = Tz + TE;
+		    TG = Tu + TF;
+		    T7l = TF - Tu;
+		    T5G = T36 + T37;
+		    T5H = T3c + T3d;
+		    T5I = T5G - T5H;
+		    T73 = T5G + T5H;
+	       }
+	       {
+		    E T38, T39, T3b, T3e;
+		    T38 = T36 - T37;
+		    T39 = To - Tt;
+		    T3a = T38 - T39;
+		    T4U = T39 + T38;
+		    T3b = Tz - TE;
+		    T3e = T3c - T3d;
+		    T3f = T3b + T3e;
+		    T4V = T3b - T3e;
+	       }
+	  }
+	  {
+	       E TM, T3i, T12, T3p, TR, T3j, TX, T3o;
+	       {
+		    E TJ, TL, TI, TK;
+		    TJ = ri[WS(ios, 2)];
+		    TL = ii[WS(ios, 2)];
+		    TI = W[2];
+		    TK = W[3];
+		    TM = FMA(TI, TJ, TK * TL);
+		    T3i = FNMS(TK, TJ, TI * TL);
+	       }
+	       {
+		    E TZ, T11, TY, T10;
+		    TZ = ri[WS(ios, 26)];
+		    T11 = ii[WS(ios, 26)];
+		    TY = W[50];
+		    T10 = W[51];
+		    T12 = FMA(TY, TZ, T10 * T11);
+		    T3p = FNMS(T10, TZ, TY * T11);
+	       }
+	       {
+		    E TO, TQ, TN, TP;
+		    TO = ri[WS(ios, 18)];
+		    TQ = ii[WS(ios, 18)];
+		    TN = W[34];
+		    TP = W[35];
+		    TR = FMA(TN, TO, TP * TQ);
+		    T3j = FNMS(TP, TO, TN * TQ);
+	       }
+	       {
+		    E TU, TW, TT, TV;
+		    TU = ri[WS(ios, 10)];
+		    TW = ii[WS(ios, 10)];
+		    TT = W[18];
+		    TV = W[19];
+		    TX = FMA(TT, TU, TV * TW);
+		    T3o = FNMS(TV, TU, TT * TW);
+	       }
+	       {
+		    E TS, T13, T5K, T5L;
+		    TS = TM + TR;
+		    T13 = TX + T12;
+		    T14 = TS + T13;
+		    T5N = TS - T13;
+		    T5K = T3i + T3j;
+		    T5L = T3o + T3p;
+		    T5M = T5K - T5L;
+		    T6E = T5K + T5L;
+	       }
+	       {
+		    E T3k, T3l, T3n, T3q;
+		    T3k = T3i - T3j;
+		    T3l = TX - T12;
+		    T3m = T3k + T3l;
+		    T4Y = T3k - T3l;
+		    T3n = TM - TR;
+		    T3q = T3o - T3p;
+		    T3r = T3n - T3q;
+		    T4Z = T3n + T3q;
+	       }
+	  }
+	  {
+	       E T19, T3t, T1p, T3A, T1e, T3u, T1k, T3z;
+	       {
+		    E T16, T18, T15, T17;
+		    T16 = ri[WS(ios, 30)];
+		    T18 = ii[WS(ios, 30)];
+		    T15 = W[58];
+		    T17 = W[59];
+		    T19 = FMA(T15, T16, T17 * T18);
+		    T3t = FNMS(T17, T16, T15 * T18);
+	       }
+	       {
+		    E T1m, T1o, T1l, T1n;
+		    T1m = ri[WS(ios, 22)];
+		    T1o = ii[WS(ios, 22)];
+		    T1l = W[42];
+		    T1n = W[43];
+		    T1p = FMA(T1l, T1m, T1n * T1o);
+		    T3A = FNMS(T1n, T1m, T1l * T1o);
+	       }
+	       {
+		    E T1b, T1d, T1a, T1c;
+		    T1b = ri[WS(ios, 14)];
+		    T1d = ii[WS(ios, 14)];
+		    T1a = W[26];
+		    T1c = W[27];
+		    T1e = FMA(T1a, T1b, T1c * T1d);
+		    T3u = FNMS(T1c, T1b, T1a * T1d);
+	       }
+	       {
+		    E T1h, T1j, T1g, T1i;
+		    T1h = ri[WS(ios, 6)];
+		    T1j = ii[WS(ios, 6)];
+		    T1g = W[10];
+		    T1i = W[11];
+		    T1k = FMA(T1g, T1h, T1i * T1j);
+		    T3z = FNMS(T1i, T1h, T1g * T1j);
+	       }
+	       {
+		    E T1f, T1q, T5Q, T5R;
+		    T1f = T19 + T1e;
+		    T1q = T1k + T1p;
+		    T1r = T1f + T1q;
+		    T5P = T1f - T1q;
+		    T5Q = T3t + T3u;
+		    T5R = T3z + T3A;
+		    T5S = T5Q - T5R;
+		    T6F = T5Q + T5R;
+	       }
+	       {
+		    E T3v, T3w, T3y, T3B;
+		    T3v = T3t - T3u;
+		    T3w = T1k - T1p;
+		    T3x = T3v + T3w;
+		    T51 = T3v - T3w;
+		    T3y = T19 - T1e;
+		    T3B = T3z - T3A;
+		    T3C = T3y - T3B;
+		    T52 = T3y + T3B;
+	       }
+	  }
+	  {
+	       E T1V, T3R, T20, T3S, T3Q, T3T, T26, T3M, T2b, T3N, T3L, T3O;
+	       {
+		    E T1S, T1U, T1R, T1T;
+		    T1S = ri[WS(ios, 5)];
+		    T1U = ii[WS(ios, 5)];
+		    T1R = W[8];
+		    T1T = W[9];
+		    T1V = FMA(T1R, T1S, T1T * T1U);
+		    T3R = FNMS(T1T, T1S, T1R * T1U);
+	       }
+	       {
+		    E T1X, T1Z, T1W, T1Y;
+		    T1X = ri[WS(ios, 21)];
+		    T1Z = ii[WS(ios, 21)];
+		    T1W = W[40];
+		    T1Y = W[41];
+		    T20 = FMA(T1W, T1X, T1Y * T1Z);
+		    T3S = FNMS(T1Y, T1X, T1W * T1Z);
+	       }
+	       T3Q = T1V - T20;
+	       T3T = T3R - T3S;
+	       {
+		    E T23, T25, T22, T24;
+		    T23 = ri[WS(ios, 29)];
+		    T25 = ii[WS(ios, 29)];
+		    T22 = W[56];
+		    T24 = W[57];
+		    T26 = FMA(T22, T23, T24 * T25);
+		    T3M = FNMS(T24, T23, T22 * T25);
+	       }
+	       {
+		    E T28, T2a, T27, T29;
+		    T28 = ri[WS(ios, 13)];
+		    T2a = ii[WS(ios, 13)];
+		    T27 = W[24];
+		    T29 = W[25];
+		    T2b = FMA(T27, T28, T29 * T2a);
+		    T3N = FNMS(T29, T28, T27 * T2a);
+	       }
+	       T3L = T26 - T2b;
+	       T3O = T3M - T3N;
+	       {
+		    E T21, T2c, T62, T63;
+		    T21 = T1V + T20;
+		    T2c = T26 + T2b;
+		    T2d = T21 + T2c;
+		    T5Z = T2c - T21;
+		    T62 = T3R + T3S;
+		    T63 = T3M + T3N;
+		    T64 = T62 - T63;
+		    T6K = T62 + T63;
+	       }
+	       {
+		    E T3P, T3U, T42, T43;
+		    T3P = T3L - T3O;
+		    T3U = T3Q + T3T;
+		    T3V = KP707106781 * (T3P - T3U);
+		    T57 = KP707106781 * (T3U + T3P);
+		    T42 = T3T - T3Q;
+		    T43 = T3L + T3O;
+		    T44 = KP707106781 * (T42 - T43);
+		    T5a = KP707106781 * (T42 + T43);
+	       }
+	  }
+	  {
+	       E T2G, T4c, T2L, T4d, T4e, T4f, T2R, T4i, T2W, T4j, T4h, T4k;
+	       {
+		    E T2D, T2F, T2C, T2E;
+		    T2D = ri[WS(ios, 3)];
+		    T2F = ii[WS(ios, 3)];
+		    T2C = W[4];
+		    T2E = W[5];
+		    T2G = FMA(T2C, T2D, T2E * T2F);
+		    T4c = FNMS(T2E, T2D, T2C * T2F);
+	       }
+	       {
+		    E T2I, T2K, T2H, T2J;
+		    T2I = ri[WS(ios, 19)];
+		    T2K = ii[WS(ios, 19)];
+		    T2H = W[36];
+		    T2J = W[37];
+		    T2L = FMA(T2H, T2I, T2J * T2K);
+		    T4d = FNMS(T2J, T2I, T2H * T2K);
+	       }
+	       T4e = T4c - T4d;
+	       T4f = T2G - T2L;
+	       {
+		    E T2O, T2Q, T2N, T2P;
+		    T2O = ri[WS(ios, 27)];
+		    T2Q = ii[WS(ios, 27)];
+		    T2N = W[52];
+		    T2P = W[53];
+		    T2R = FMA(T2N, T2O, T2P * T2Q);
+		    T4i = FNMS(T2P, T2O, T2N * T2Q);
+	       }
+	       {
+		    E T2T, T2V, T2S, T2U;
+		    T2T = ri[WS(ios, 11)];
+		    T2V = ii[WS(ios, 11)];
+		    T2S = W[20];
+		    T2U = W[21];
+		    T2W = FMA(T2S, T2T, T2U * T2V);
+		    T4j = FNMS(T2U, T2T, T2S * T2V);
+	       }
+	       T4h = T2R - T2W;
+	       T4k = T4i - T4j;
+	       {
+		    E T2M, T2X, T68, T69;
+		    T2M = T2G + T2L;
+		    T2X = T2R + T2W;
+		    T2Y = T2M + T2X;
+		    T6f = T2X - T2M;
+		    T68 = T4c + T4d;
+		    T69 = T4i + T4j;
+		    T6a = T68 - T69;
+		    T6P = T68 + T69;
+	       }
+	       {
+		    E T4g, T4l, T4t, T4u;
+		    T4g = T4e - T4f;
+		    T4l = T4h + T4k;
+		    T4m = KP707106781 * (T4g - T4l);
+		    T5h = KP707106781 * (T4g + T4l);
+		    T4t = T4h - T4k;
+		    T4u = T4f + T4e;
+		    T4v = KP707106781 * (T4t - T4u);
+		    T5e = KP707106781 * (T4u + T4t);
+	       }
+	  }
+	  {
+	       E T1t, T6X, T7a, T7c, T30, T7b, T70, T71;
+	       {
+		    E TH, T1s, T72, T79;
+		    TH = Tj + TG;
+		    T1s = T14 + T1r;
+		    T1t = TH + T1s;
+		    T6X = TH - T1s;
+		    T72 = T6E + T6F;
+		    T79 = T73 + T78;
+		    T7a = T72 + T79;
+		    T7c = T79 - T72;
+	       }
+	       {
+		    E T2e, T2Z, T6Y, T6Z;
+		    T2e = T1Q + T2d;
+		    T2Z = T2B + T2Y;
+		    T30 = T2e + T2Z;
+		    T7b = T2Z - T2e;
+		    T6Y = T6J + T6K;
+		    T6Z = T6O + T6P;
+		    T70 = T6Y - T6Z;
+		    T71 = T6Y + T6Z;
+	       }
+	       ri[WS(ios, 16)] = T1t - T30;
+	       ii[WS(ios, 16)] = T7a - T71;
+	       ri[0] = T1t + T30;
+	       ii[0] = T71 + T7a;
+	       ri[WS(ios, 24)] = T6X - T70;
+	       ii[WS(ios, 24)] = T7c - T7b;
+	       ri[WS(ios, 8)] = T6X + T70;
+	       ii[WS(ios, 8)] = T7b + T7c;
+	  }
+	  {
+	       E T6H, T6T, T7g, T7i, T6M, T6U, T6R, T6V;
+	       {
+		    E T6D, T6G, T7e, T7f;
+		    T6D = Tj - TG;
+		    T6G = T6E - T6F;
+		    T6H = T6D + T6G;
+		    T6T = T6D - T6G;
+		    T7e = T1r - T14;
+		    T7f = T78 - T73;
+		    T7g = T7e + T7f;
+		    T7i = T7f - T7e;
+	       }
+	       {
+		    E T6I, T6L, T6N, T6Q;
+		    T6I = T1Q - T2d;
+		    T6L = T6J - T6K;
+		    T6M = T6I + T6L;
+		    T6U = T6L - T6I;
+		    T6N = T2B - T2Y;
+		    T6Q = T6O - T6P;
+		    T6R = T6N - T6Q;
+		    T6V = T6N + T6Q;
+	       }
+	       {
+		    E T6S, T7d, T6W, T7h;
+		    T6S = KP707106781 * (T6M + T6R);
+		    ri[WS(ios, 20)] = T6H - T6S;
+		    ri[WS(ios, 4)] = T6H + T6S;
+		    T7d = KP707106781 * (T6U + T6V);
+		    ii[WS(ios, 4)] = T7d + T7g;
+		    ii[WS(ios, 20)] = T7g - T7d;
+		    T6W = KP707106781 * (T6U - T6V);
+		    ri[WS(ios, 28)] = T6T - T6W;
+		    ri[WS(ios, 12)] = T6T + T6W;
+		    T7h = KP707106781 * (T6R - T6M);
+		    ii[WS(ios, 12)] = T7h + T7i;
+		    ii[WS(ios, 28)] = T7i - T7h;
+	       }
+	  }
+	  {
+	       E T5J, T7n, T7t, T6n, T5U, T7k, T6x, T6B, T6q, T7s, T66, T6k, T6u, T6A, T6h;
+	       E T6l;
+	       {
+		    E T5O, T5T, T60, T65;
+		    T5J = T5F - T5I;
+		    T7n = T7l + T7m;
+		    T7t = T7m - T7l;
+		    T6n = T5F + T5I;
+		    T5O = T5M - T5N;
+		    T5T = T5P + T5S;
+		    T5U = KP707106781 * (T5O - T5T);
+		    T7k = KP707106781 * (T5O + T5T);
+		    {
+			 E T6v, T6w, T6o, T6p;
+			 T6v = T67 + T6a;
+			 T6w = T6e + T6f;
+			 T6x = FNMS(KP382683432, T6w, KP923879532 * T6v);
+			 T6B = FMA(KP923879532, T6w, KP382683432 * T6v);
+			 T6o = T5N + T5M;
+			 T6p = T5P - T5S;
+			 T6q = KP707106781 * (T6o + T6p);
+			 T7s = KP707106781 * (T6p - T6o);
+		    }
+		    T60 = T5Y - T5Z;
+		    T65 = T61 - T64;
+		    T66 = FMA(KP923879532, T60, KP382683432 * T65);
+		    T6k = FNMS(KP923879532, T65, KP382683432 * T60);
+		    {
+			 E T6s, T6t, T6b, T6g;
+			 T6s = T5Y + T5Z;
+			 T6t = T61 + T64;
+			 T6u = FMA(KP382683432, T6s, KP923879532 * T6t);
+			 T6A = FNMS(KP382683432, T6t, KP923879532 * T6s);
+			 T6b = T67 - T6a;
+			 T6g = T6e - T6f;
+			 T6h = FNMS(KP923879532, T6g, KP382683432 * T6b);
+			 T6l = FMA(KP382683432, T6g, KP923879532 * T6b);
+		    }
+	       }
+	       {
+		    E T5V, T6i, T7r, T7u;
+		    T5V = T5J + T5U;
+		    T6i = T66 + T6h;
+		    ri[WS(ios, 22)] = T5V - T6i;
+		    ri[WS(ios, 6)] = T5V + T6i;
+		    T7r = T6k + T6l;
+		    T7u = T7s + T7t;
+		    ii[WS(ios, 6)] = T7r + T7u;
+		    ii[WS(ios, 22)] = T7u - T7r;
+	       }
+	       {
+		    E T6j, T6m, T7v, T7w;
+		    T6j = T5J - T5U;
+		    T6m = T6k - T6l;
+		    ri[WS(ios, 30)] = T6j - T6m;
+		    ri[WS(ios, 14)] = T6j + T6m;
+		    T7v = T6h - T66;
+		    T7w = T7t - T7s;
+		    ii[WS(ios, 14)] = T7v + T7w;
+		    ii[WS(ios, 30)] = T7w - T7v;
+	       }
+	       {
+		    E T6r, T6y, T7j, T7o;
+		    T6r = T6n + T6q;
+		    T6y = T6u + T6x;
+		    ri[WS(ios, 18)] = T6r - T6y;
+		    ri[WS(ios, 2)] = T6r + T6y;
+		    T7j = T6A + T6B;
+		    T7o = T7k + T7n;
+		    ii[WS(ios, 2)] = T7j + T7o;
+		    ii[WS(ios, 18)] = T7o - T7j;
+	       }
+	       {
+		    E T6z, T6C, T7p, T7q;
+		    T6z = T6n - T6q;
+		    T6C = T6A - T6B;
+		    ri[WS(ios, 26)] = T6z - T6C;
+		    ri[WS(ios, 10)] = T6z + T6C;
+		    T7p = T6x - T6u;
+		    T7q = T7n - T7k;
+		    ii[WS(ios, 10)] = T7p + T7q;
+		    ii[WS(ios, 26)] = T7q - T7p;
+	       }
+	  }
+	  {
+	       E T3h, T4D, T7R, T7X, T3E, T7O, T4N, T4R, T46, T4A, T4G, T7W, T4K, T4Q, T4x;
+	       E T4B, T3g, T7P;
+	       T3g = KP707106781 * (T3a - T3f);
+	       T3h = T35 - T3g;
+	       T4D = T35 + T3g;
+	       T7P = KP707106781 * (T4V - T4U);
+	       T7R = T7P + T7Q;
+	       T7X = T7Q - T7P;
+	       {
+		    E T3s, T3D, T4L, T4M;
+		    T3s = FNMS(KP923879532, T3r, KP382683432 * T3m);
+		    T3D = FMA(KP382683432, T3x, KP923879532 * T3C);
+		    T3E = T3s - T3D;
+		    T7O = T3s + T3D;
+		    T4L = T4b + T4m;
+		    T4M = T4s + T4v;
+		    T4N = FNMS(KP555570233, T4M, KP831469612 * T4L);
+		    T4R = FMA(KP831469612, T4M, KP555570233 * T4L);
+	       }
+	       {
+		    E T3W, T45, T4E, T4F;
+		    T3W = T3K - T3V;
+		    T45 = T41 - T44;
+		    T46 = FMA(KP980785280, T3W, KP195090322 * T45);
+		    T4A = FNMS(KP980785280, T45, KP195090322 * T3W);
+		    T4E = FMA(KP923879532, T3m, KP382683432 * T3r);
+		    T4F = FNMS(KP923879532, T3x, KP382683432 * T3C);
+		    T4G = T4E + T4F;
+		    T7W = T4F - T4E;
+	       }
+	       {
+		    E T4I, T4J, T4n, T4w;
+		    T4I = T3K + T3V;
+		    T4J = T41 + T44;
+		    T4K = FMA(KP555570233, T4I, KP831469612 * T4J);
+		    T4Q = FNMS(KP555570233, T4J, KP831469612 * T4I);
+		    T4n = T4b - T4m;
+		    T4w = T4s - T4v;
+		    T4x = FNMS(KP980785280, T4w, KP195090322 * T4n);
+		    T4B = FMA(KP195090322, T4w, KP980785280 * T4n);
+	       }
+	       {
+		    E T3F, T4y, T7V, T7Y;
+		    T3F = T3h + T3E;
+		    T4y = T46 + T4x;
+		    ri[WS(ios, 23)] = T3F - T4y;
+		    ri[WS(ios, 7)] = T3F + T4y;
+		    T7V = T4A + T4B;
+		    T7Y = T7W + T7X;
+		    ii[WS(ios, 7)] = T7V + T7Y;
+		    ii[WS(ios, 23)] = T7Y - T7V;
+	       }
+	       {
+		    E T4z, T4C, T7Z, T80;
+		    T4z = T3h - T3E;
+		    T4C = T4A - T4B;
+		    ri[WS(ios, 31)] = T4z - T4C;
+		    ri[WS(ios, 15)] = T4z + T4C;
+		    T7Z = T4x - T46;
+		    T80 = T7X - T7W;
+		    ii[WS(ios, 15)] = T7Z + T80;
+		    ii[WS(ios, 31)] = T80 - T7Z;
+	       }
+	       {
+		    E T4H, T4O, T7N, T7S;
+		    T4H = T4D + T4G;
+		    T4O = T4K + T4N;
+		    ri[WS(ios, 19)] = T4H - T4O;
+		    ri[WS(ios, 3)] = T4H + T4O;
+		    T7N = T4Q + T4R;
+		    T7S = T7O + T7R;
+		    ii[WS(ios, 3)] = T7N + T7S;
+		    ii[WS(ios, 19)] = T7S - T7N;
+	       }
+	       {
+		    E T4P, T4S, T7T, T7U;
+		    T4P = T4D - T4G;
+		    T4S = T4Q - T4R;
+		    ri[WS(ios, 27)] = T4P - T4S;
+		    ri[WS(ios, 11)] = T4P + T4S;
+		    T7T = T4N - T4K;
+		    T7U = T7R - T7O;
+		    ii[WS(ios, 11)] = T7T + T7U;
+		    ii[WS(ios, 27)] = T7U - T7T;
+	       }
+	  }
+	  {
+	       E T4X, T5p, T7D, T7J, T54, T7y, T5z, T5D, T5c, T5m, T5s, T7I, T5w, T5C, T5j;
+	       E T5n, T4W, T7z;
+	       T4W = KP707106781 * (T4U + T4V);
+	       T4X = T4T - T4W;
+	       T5p = T4T + T4W;
+	       T7z = KP707106781 * (T3a + T3f);
+	       T7D = T7z + T7C;
+	       T7J = T7C - T7z;
+	       {
+		    E T50, T53, T5x, T5y;
+		    T50 = FNMS(KP382683432, T4Z, KP923879532 * T4Y);
+		    T53 = FMA(KP923879532, T51, KP382683432 * T52);
+		    T54 = T50 - T53;
+		    T7y = T50 + T53;
+		    T5x = T5d + T5e;
+		    T5y = T5g + T5h;
+		    T5z = FNMS(KP195090322, T5y, KP980785280 * T5x);
+		    T5D = FMA(KP195090322, T5x, KP980785280 * T5y);
+	       }
+	       {
+		    E T58, T5b, T5q, T5r;
+		    T58 = T56 - T57;
+		    T5b = T59 - T5a;
+		    T5c = FMA(KP555570233, T58, KP831469612 * T5b);
+		    T5m = FNMS(KP831469612, T58, KP555570233 * T5b);
+		    T5q = FMA(KP382683432, T4Y, KP923879532 * T4Z);
+		    T5r = FNMS(KP382683432, T51, KP923879532 * T52);
+		    T5s = T5q + T5r;
+		    T7I = T5r - T5q;
+	       }
+	       {
+		    E T5u, T5v, T5f, T5i;
+		    T5u = T56 + T57;
+		    T5v = T59 + T5a;
+		    T5w = FMA(KP980785280, T5u, KP195090322 * T5v);
+		    T5C = FNMS(KP195090322, T5u, KP980785280 * T5v);
+		    T5f = T5d - T5e;
+		    T5i = T5g - T5h;
+		    T5j = FNMS(KP831469612, T5i, KP555570233 * T5f);
+		    T5n = FMA(KP831469612, T5f, KP555570233 * T5i);
+	       }
+	       {
+		    E T55, T5k, T7H, T7K;
+		    T55 = T4X + T54;
+		    T5k = T5c + T5j;
+		    ri[WS(ios, 21)] = T55 - T5k;
+		    ri[WS(ios, 5)] = T55 + T5k;
+		    T7H = T5m + T5n;
+		    T7K = T7I + T7J;
+		    ii[WS(ios, 5)] = T7H + T7K;
+		    ii[WS(ios, 21)] = T7K - T7H;
+	       }
+	       {
+		    E T5l, T5o, T7L, T7M;
+		    T5l = T4X - T54;
+		    T5o = T5m - T5n;
+		    ri[WS(ios, 29)] = T5l - T5o;
+		    ri[WS(ios, 13)] = T5l + T5o;
+		    T7L = T5j - T5c;
+		    T7M = T7J - T7I;
+		    ii[WS(ios, 13)] = T7L + T7M;
+		    ii[WS(ios, 29)] = T7M - T7L;
+	       }
+	       {
+		    E T5t, T5A, T7x, T7E;
+		    T5t = T5p + T5s;
+		    T5A = T5w + T5z;
+		    ri[WS(ios, 17)] = T5t - T5A;
+		    ri[WS(ios, 1)] = T5t + T5A;
+		    T7x = T5C + T5D;
+		    T7E = T7y + T7D;
+		    ii[WS(ios, 1)] = T7x + T7E;
+		    ii[WS(ios, 17)] = T7E - T7x;
+	       }
+	       {
+		    E T5B, T5E, T7F, T7G;
+		    T5B = T5p - T5s;
+		    T5E = T5C - T5D;
+		    ri[WS(ios, 25)] = T5B - T5E;
+		    ri[WS(ios, 9)] = T5B + T5E;
+		    T7F = T5z - T5w;
+		    T7G = T7D - T7y;
+		    ii[WS(ios, 9)] = T7F + T7G;
+		    ii[WS(ios, 25)] = T7G - T7F;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 32},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 32, "t1_32", twinstr, {340, 114, 94, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_32) (planner *p) {
+     X(kdft_dit_register) (p, t1_32, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_4.c b/src/fftw3/dft/codelets/standard/t1_4.c
new file mode 100644
index 0000000..cc869e4
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_4.c
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:52 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 4 -name t1_4 -include t.h */
+
+/*
+ * This function contains 22 FP additions, 12 FP multiplications,
+ * (or, 16 additions, 6 multiplications, 6 fused multiply/add),
+ * 13 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_4.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_4.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_4.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_4(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 6) {
+	  E T1, Tp, T6, To, Tc, Tk, Th, Tl;
+	  T1 = ri[0];
+	  Tp = ii[0];
+	  {
+	       E T3, T5, T2, T4;
+	       T3 = ri[WS(ios, 2)];
+	       T5 = ii[WS(ios, 2)];
+	       T2 = W[2];
+	       T4 = W[3];
+	       T6 = FMA(T2, T3, T4 * T5);
+	       To = FNMS(T4, T3, T2 * T5);
+	  }
+	  {
+	       E T9, Tb, T8, Ta;
+	       T9 = ri[WS(ios, 1)];
+	       Tb = ii[WS(ios, 1)];
+	       T8 = W[0];
+	       Ta = W[1];
+	       Tc = FMA(T8, T9, Ta * Tb);
+	       Tk = FNMS(Ta, T9, T8 * Tb);
+	  }
+	  {
+	       E Te, Tg, Td, Tf;
+	       Te = ri[WS(ios, 3)];
+	       Tg = ii[WS(ios, 3)];
+	       Td = W[4];
+	       Tf = W[5];
+	       Th = FMA(Td, Te, Tf * Tg);
+	       Tl = FNMS(Tf, Te, Td * Tg);
+	  }
+	  {
+	       E T7, Ti, Tn, Tq;
+	       T7 = T1 + T6;
+	       Ti = Tc + Th;
+	       ri[WS(ios, 2)] = T7 - Ti;
+	       ri[0] = T7 + Ti;
+	       Tn = Tk + Tl;
+	       Tq = To + Tp;
+	       ii[0] = Tn + Tq;
+	       ii[WS(ios, 2)] = Tq - Tn;
+	  }
+	  {
+	       E Tj, Tm, Tr, Ts;
+	       Tj = T1 - T6;
+	       Tm = Tk - Tl;
+	       ri[WS(ios, 3)] = Tj - Tm;
+	       ri[WS(ios, 1)] = Tj + Tm;
+	       Tr = Tp - To;
+	       Ts = Tc - Th;
+	       ii[WS(ios, 1)] = Tr - Ts;
+	       ii[WS(ios, 3)] = Ts + Tr;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 4},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 4, "t1_4", twinstr, {16, 6, 6, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_4) (planner *p) {
+     X(kdft_dit_register) (p, t1_4, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_5.c b/src/fftw3/dft/codelets/standard/t1_5.c
new file mode 100644
index 0000000..87feed7
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_5.c
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:52 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 5 -name t1_5 -include t.h */
+
+/*
+ * This function contains 40 FP additions, 28 FP multiplications,
+ * (or, 26 additions, 14 multiplications, 14 fused multiply/add),
+ * 29 stack variables, and 20 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_5.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_5.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_5.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_5(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 8) {
+	  E T1, TE, Tu, Tx, TJ, TI, TB, TC, TD, Tc, Tn, To;
+	  T1 = ri[0];
+	  TE = ii[0];
+	  {
+	       E T6, Ts, Tm, Tw, Tb, Tt, Th, Tv;
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = ri[WS(ios, 1)];
+		    T5 = ii[WS(ios, 1)];
+		    T2 = W[0];
+		    T4 = W[1];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    Ts = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E Tj, Tl, Ti, Tk;
+		    Tj = ri[WS(ios, 3)];
+		    Tl = ii[WS(ios, 3)];
+		    Ti = W[4];
+		    Tk = W[5];
+		    Tm = FMA(Ti, Tj, Tk * Tl);
+		    Tw = FNMS(Tk, Tj, Ti * Tl);
+	       }
+	       {
+		    E T8, Ta, T7, T9;
+		    T8 = ri[WS(ios, 4)];
+		    Ta = ii[WS(ios, 4)];
+		    T7 = W[6];
+		    T9 = W[7];
+		    Tb = FMA(T7, T8, T9 * Ta);
+		    Tt = FNMS(T9, T8, T7 * Ta);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = ri[WS(ios, 2)];
+		    Tg = ii[WS(ios, 2)];
+		    Td = W[2];
+		    Tf = W[3];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    Tv = FNMS(Tf, Te, Td * Tg);
+	       }
+	       Tu = Ts - Tt;
+	       Tx = Tv - Tw;
+	       TJ = Th - Tm;
+	       TI = T6 - Tb;
+	       TB = Ts + Tt;
+	       TC = Tv + Tw;
+	       TD = TB + TC;
+	       Tc = T6 + Tb;
+	       Tn = Th + Tm;
+	       To = Tc + Tn;
+	  }
+	  ri[0] = T1 + To;
+	  ii[0] = TD + TE;
+	  {
+	       E Ty, TA, Tr, Tz, Tp, Tq;
+	       Ty = FMA(KP951056516, Tu, KP587785252 * Tx);
+	       TA = FNMS(KP587785252, Tu, KP951056516 * Tx);
+	       Tp = KP559016994 * (Tc - Tn);
+	       Tq = FNMS(KP250000000, To, T1);
+	       Tr = Tp + Tq;
+	       Tz = Tq - Tp;
+	       ri[WS(ios, 4)] = Tr - Ty;
+	       ri[WS(ios, 3)] = Tz + TA;
+	       ri[WS(ios, 1)] = Tr + Ty;
+	       ri[WS(ios, 2)] = Tz - TA;
+	  }
+	  {
+	       E TK, TL, TH, TM, TF, TG;
+	       TK = FMA(KP951056516, TI, KP587785252 * TJ);
+	       TL = FNMS(KP587785252, TI, KP951056516 * TJ);
+	       TF = KP559016994 * (TB - TC);
+	       TG = FNMS(KP250000000, TD, TE);
+	       TH = TF + TG;
+	       TM = TG - TF;
+	       ii[WS(ios, 1)] = TH - TK;
+	       ii[WS(ios, 3)] = TM - TL;
+	       ii[WS(ios, 4)] = TK + TH;
+	       ii[WS(ios, 2)] = TL + TM;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 5},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 5, "t1_5", twinstr, {26, 14, 14, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_5) (planner *p) {
+     X(kdft_dit_register) (p, t1_5, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_6.c b/src/fftw3/dft/codelets/standard/t1_6.c
new file mode 100644
index 0000000..d474acd
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_6.c
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:55 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 6 -name t1_6 -include t.h */
+
+/*
+ * This function contains 46 FP additions, 28 FP multiplications,
+ * (or, 32 additions, 14 multiplications, 14 fused multiply/add),
+ * 23 stack variables, and 24 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_6.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_6.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_6.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_6(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 10) {
+	  E T7, TS, Tv, TO, Tt, TJ, Tx, TF, Ti, TI, Tw, TC;
+	  {
+	       E T1, TN, T6, TM;
+	       T1 = ri[0];
+	       TN = ii[0];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = ri[WS(ios, 3)];
+		    T5 = ii[WS(ios, 3)];
+		    T2 = W[4];
+		    T4 = W[5];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    TM = FNMS(T4, T3, T2 * T5);
+	       }
+	       T7 = T1 - T6;
+	       TS = TN - TM;
+	       Tv = T1 + T6;
+	       TO = TM + TN;
+	  }
+	  {
+	       E Tn, TD, Ts, TE;
+	       {
+		    E Tk, Tm, Tj, Tl;
+		    Tk = ri[WS(ios, 4)];
+		    Tm = ii[WS(ios, 4)];
+		    Tj = W[6];
+		    Tl = W[7];
+		    Tn = FMA(Tj, Tk, Tl * Tm);
+		    TD = FNMS(Tl, Tk, Tj * Tm);
+	       }
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = ri[WS(ios, 1)];
+		    Tr = ii[WS(ios, 1)];
+		    To = W[0];
+		    Tq = W[1];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    TE = FNMS(Tq, Tp, To * Tr);
+	       }
+	       Tt = Tn - Ts;
+	       TJ = TD + TE;
+	       Tx = Tn + Ts;
+	       TF = TD - TE;
+	  }
+	  {
+	       E Tc, TA, Th, TB;
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = ri[WS(ios, 2)];
+		    Tb = ii[WS(ios, 2)];
+		    T8 = W[2];
+		    Ta = W[3];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    TA = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = ri[WS(ios, 5)];
+		    Tg = ii[WS(ios, 5)];
+		    Td = W[8];
+		    Tf = W[9];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    TB = FNMS(Tf, Te, Td * Tg);
+	       }
+	       Ti = Tc - Th;
+	       TI = TA + TB;
+	       Tw = Tc + Th;
+	       TC = TA - TB;
+	  }
+	  {
+	       E TG, Tu, Tz, TR, TT, TU;
+	       TG = KP866025403 * (TC - TF);
+	       Tu = Ti + Tt;
+	       Tz = FNMS(KP500000000, Tu, T7);
+	       ri[WS(ios, 3)] = T7 + Tu;
+	       ri[WS(ios, 1)] = Tz + TG;
+	       ri[WS(ios, 5)] = Tz - TG;
+	       TR = KP866025403 * (Tt - Ti);
+	       TT = TC + TF;
+	       TU = FNMS(KP500000000, TT, TS);
+	       ii[WS(ios, 1)] = TR + TU;
+	       ii[WS(ios, 3)] = TT + TS;
+	       ii[WS(ios, 5)] = TU - TR;
+	  }
+	  {
+	       E TK, Ty, TH, TQ, TL, TP;
+	       TK = KP866025403 * (TI - TJ);
+	       Ty = Tw + Tx;
+	       TH = FNMS(KP500000000, Ty, Tv);
+	       ri[0] = Tv + Ty;
+	       ri[WS(ios, 4)] = TH + TK;
+	       ri[WS(ios, 2)] = TH - TK;
+	       TQ = KP866025403 * (Tx - Tw);
+	       TL = TI + TJ;
+	       TP = FNMS(KP500000000, TL, TO);
+	       ii[0] = TL + TO;
+	       ii[WS(ios, 4)] = TQ + TP;
+	       ii[WS(ios, 2)] = TP - TQ;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 6},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 6, "t1_6", twinstr, {32, 14, 14, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_6) (planner *p) {
+     X(kdft_dit_register) (p, t1_6, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_64.c b/src/fftw3/dft/codelets/standard/t1_64.c
new file mode 100644
index 0000000..a03697b
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_64.c
@@ -0,0 +1,2001 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:08 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 64 -name t1_64 -include t.h */
+
+/*
+ * This function contains 1038 FP additions, 500 FP multiplications,
+ * (or, 808 additions, 270 multiplications, 230 fused multiply/add),
+ * 176 stack variables, and 256 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_64.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_64.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_64.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_64(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP471396736, +0.471396736825997648556387625905254377657460319);
+     DK(KP881921264, +0.881921264348355029712756863660388349508442621);
+     DK(KP290284677, +0.290284677254462367636192375817395274691476278);
+     DK(KP956940335, +0.956940335732208864935797886980269969482849206);
+     DK(KP634393284, +0.634393284163645498215171613225493370675687095);
+     DK(KP773010453, +0.773010453362736960810906609758469800971041293);
+     DK(KP098017140, +0.098017140329560601994195563888641845861136673);
+     DK(KP995184726, +0.995184726672196886244836953109479921575474869);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 126) {
+	  E Tj, TcL, ThT, Tin, T6b, Taz, TgT, Thn, TG, Thm, TcO, TgO, T6m, ThQ, TaC;
+	  E Tim, T14, Tfq, T6y, T9O, TaG, Tc0, TcU, TeE, T1r, Tfr, T6J, T9P, TaJ, Tc1;
+	  E TcZ, TeF, T1Q, T2d, Tfx, Tfu, Tfv, Tfw, T6Q, TaM, Tdb, TeJ, T71, TaQ, T7a;
+	  E TaN, Td6, TeI, T77, TaP, T2B, T2Y, Tfz, TfA, TfB, TfC, T7h, TaW, Tdm, TeM;
+	  E T7s, TaU, T7B, TaX, Tdh, TeL, T7y, TaT, T5j, TfR, Tec, Tf0, TfY, Tgy, T8D;
+	  E Tbl, T8O, Tbx, T9l, Tbm, TdV, TeX, T9i, Tbw, T3M, TfL, TdL, TeQ, TfI, Tgt;
+	  E T7K, Tb2, T7V, Tbe, T8s, Tb3, Tdu, TeT, T8p, Tbd, T4x, TfJ, TdE, TdM, TfO;
+	  E Tgu, T87, T8v, T8i, T8u, Tba, Tbg, Tdz, TdN, Tb7, Tbh, T64, TfZ, Te5, Ted;
+	  E TfU, Tgz, T90, T9o, T9b, T9n, Tbt, Tbz, Te0, Tee, Tbq, TbA;
+	  {
+	       E T1, TgR, T6, TgQ, Tc, T68, Th, T69;
+	       T1 = ri[0];
+	       TgR = ii[0];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = ri[WS(ios, 32)];
+		    T5 = ii[WS(ios, 32)];
+		    T2 = W[62];
+		    T4 = W[63];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    TgQ = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = ri[WS(ios, 16)];
+		    Tb = ii[WS(ios, 16)];
+		    T8 = W[30];
+		    Ta = W[31];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    T68 = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = ri[WS(ios, 48)];
+		    Tg = ii[WS(ios, 48)];
+		    Td = W[94];
+		    Tf = W[95];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    T69 = FNMS(Tf, Te, Td * Tg);
+	       }
+	       {
+		    E T7, Ti, ThR, ThS;
+		    T7 = T1 + T6;
+		    Ti = Tc + Th;
+		    Tj = T7 + Ti;
+		    TcL = T7 - Ti;
+		    ThR = TgR - TgQ;
+		    ThS = Tc - Th;
+		    ThT = ThR - ThS;
+		    Tin = ThS + ThR;
+	       }
+	       {
+		    E T67, T6a, TgP, TgS;
+		    T67 = T1 - T6;
+		    T6a = T68 - T69;
+		    T6b = T67 - T6a;
+		    Taz = T67 + T6a;
+		    TgP = T68 + T69;
+		    TgS = TgQ + TgR;
+		    TgT = TgP + TgS;
+		    Thn = TgS - TgP;
+	       }
+	  }
+	  {
+	       E To, T6c, Tt, T6d, T6e, T6f, Tz, T6i, TE, T6j, T6h, T6k;
+	       {
+		    E Tl, Tn, Tk, Tm;
+		    Tl = ri[WS(ios, 8)];
+		    Tn = ii[WS(ios, 8)];
+		    Tk = W[14];
+		    Tm = W[15];
+		    To = FMA(Tk, Tl, Tm * Tn);
+		    T6c = FNMS(Tm, Tl, Tk * Tn);
+	       }
+	       {
+		    E Tq, Ts, Tp, Tr;
+		    Tq = ri[WS(ios, 40)];
+		    Ts = ii[WS(ios, 40)];
+		    Tp = W[78];
+		    Tr = W[79];
+		    Tt = FMA(Tp, Tq, Tr * Ts);
+		    T6d = FNMS(Tr, Tq, Tp * Ts);
+	       }
+	       T6e = T6c - T6d;
+	       T6f = To - Tt;
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = ri[WS(ios, 56)];
+		    Ty = ii[WS(ios, 56)];
+		    Tv = W[110];
+		    Tx = W[111];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T6i = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = ri[WS(ios, 24)];
+		    TD = ii[WS(ios, 24)];
+		    TA = W[46];
+		    TC = W[47];
+		    TE = FMA(TA, TB, TC * TD);
+		    T6j = FNMS(TC, TB, TA * TD);
+	       }
+	       T6h = Tz - TE;
+	       T6k = T6i - T6j;
+	       {
+		    E Tu, TF, TcM, TcN;
+		    Tu = To + Tt;
+		    TF = Tz + TE;
+		    TG = Tu + TF;
+		    Thm = TF - Tu;
+		    TcM = T6c + T6d;
+		    TcN = T6i + T6j;
+		    TcO = TcM - TcN;
+		    TgO = TcM + TcN;
+	       }
+	       {
+		    E T6g, T6l, TaA, TaB;
+		    T6g = T6e - T6f;
+		    T6l = T6h + T6k;
+		    T6m = KP707106781 * (T6g - T6l);
+		    ThQ = KP707106781 * (T6g + T6l);
+		    TaA = T6f + T6e;
+		    TaB = T6h - T6k;
+		    TaC = KP707106781 * (TaA + TaB);
+		    Tim = KP707106781 * (TaB - TaA);
+	       }
+	  }
+	  {
+	       E TS, TcQ, T6q, T6t, T13, TcR, T6r, T6w, T6s, T6x;
+	       {
+		    E TM, T6o, TR, T6p;
+		    {
+			 E TJ, TL, TI, TK;
+			 TJ = ri[WS(ios, 4)];
+			 TL = ii[WS(ios, 4)];
+			 TI = W[6];
+			 TK = W[7];
+			 TM = FMA(TI, TJ, TK * TL);
+			 T6o = FNMS(TK, TJ, TI * TL);
+		    }
+		    {
+			 E TO, TQ, TN, TP;
+			 TO = ri[WS(ios, 36)];
+			 TQ = ii[WS(ios, 36)];
+			 TN = W[70];
+			 TP = W[71];
+			 TR = FMA(TN, TO, TP * TQ);
+			 T6p = FNMS(TP, TO, TN * TQ);
+		    }
+		    TS = TM + TR;
+		    TcQ = T6o + T6p;
+		    T6q = T6o - T6p;
+		    T6t = TM - TR;
+	       }
+	       {
+		    E TX, T6u, T12, T6v;
+		    {
+			 E TU, TW, TT, TV;
+			 TU = ri[WS(ios, 20)];
+			 TW = ii[WS(ios, 20)];
+			 TT = W[38];
+			 TV = W[39];
+			 TX = FMA(TT, TU, TV * TW);
+			 T6u = FNMS(TV, TU, TT * TW);
+		    }
+		    {
+			 E TZ, T11, TY, T10;
+			 TZ = ri[WS(ios, 52)];
+			 T11 = ii[WS(ios, 52)];
+			 TY = W[102];
+			 T10 = W[103];
+			 T12 = FMA(TY, TZ, T10 * T11);
+			 T6v = FNMS(T10, TZ, TY * T11);
+		    }
+		    T13 = TX + T12;
+		    TcR = T6u + T6v;
+		    T6r = TX - T12;
+		    T6w = T6u - T6v;
+	       }
+	       T14 = TS + T13;
+	       Tfq = TcQ + TcR;
+	       T6s = T6q + T6r;
+	       T6x = T6t - T6w;
+	       T6y = FNMS(KP923879532, T6x, KP382683432 * T6s);
+	       T9O = FMA(KP923879532, T6s, KP382683432 * T6x);
+	       {
+		    E TaE, TaF, TcS, TcT;
+		    TaE = T6q - T6r;
+		    TaF = T6t + T6w;
+		    TaG = FNMS(KP382683432, TaF, KP923879532 * TaE);
+		    Tc0 = FMA(KP382683432, TaE, KP923879532 * TaF);
+		    TcS = TcQ - TcR;
+		    TcT = TS - T13;
+		    TcU = TcS - TcT;
+		    TeE = TcT + TcS;
+	       }
+	  }
+	  {
+	       E T1f, TcW, T6B, T6E, T1q, TcX, T6C, T6H, T6D, T6I;
+	       {
+		    E T19, T6z, T1e, T6A;
+		    {
+			 E T16, T18, T15, T17;
+			 T16 = ri[WS(ios, 60)];
+			 T18 = ii[WS(ios, 60)];
+			 T15 = W[118];
+			 T17 = W[119];
+			 T19 = FMA(T15, T16, T17 * T18);
+			 T6z = FNMS(T17, T16, T15 * T18);
+		    }
+		    {
+			 E T1b, T1d, T1a, T1c;
+			 T1b = ri[WS(ios, 28)];
+			 T1d = ii[WS(ios, 28)];
+			 T1a = W[54];
+			 T1c = W[55];
+			 T1e = FMA(T1a, T1b, T1c * T1d);
+			 T6A = FNMS(T1c, T1b, T1a * T1d);
+		    }
+		    T1f = T19 + T1e;
+		    TcW = T6z + T6A;
+		    T6B = T6z - T6A;
+		    T6E = T19 - T1e;
+	       }
+	       {
+		    E T1k, T6F, T1p, T6G;
+		    {
+			 E T1h, T1j, T1g, T1i;
+			 T1h = ri[WS(ios, 12)];
+			 T1j = ii[WS(ios, 12)];
+			 T1g = W[22];
+			 T1i = W[23];
+			 T1k = FMA(T1g, T1h, T1i * T1j);
+			 T6F = FNMS(T1i, T1h, T1g * T1j);
+		    }
+		    {
+			 E T1m, T1o, T1l, T1n;
+			 T1m = ri[WS(ios, 44)];
+			 T1o = ii[WS(ios, 44)];
+			 T1l = W[86];
+			 T1n = W[87];
+			 T1p = FMA(T1l, T1m, T1n * T1o);
+			 T6G = FNMS(T1n, T1m, T1l * T1o);
+		    }
+		    T1q = T1k + T1p;
+		    TcX = T6F + T6G;
+		    T6C = T1k - T1p;
+		    T6H = T6F - T6G;
+	       }
+	       T1r = T1f + T1q;
+	       Tfr = TcW + TcX;
+	       T6D = T6B + T6C;
+	       T6I = T6E - T6H;
+	       T6J = FMA(KP382683432, T6D, KP923879532 * T6I);
+	       T9P = FNMS(KP923879532, T6D, KP382683432 * T6I);
+	       {
+		    E TaH, TaI, TcV, TcY;
+		    TaH = T6B - T6C;
+		    TaI = T6E + T6H;
+		    TaJ = FMA(KP923879532, TaH, KP382683432 * TaI);
+		    Tc1 = FNMS(KP382683432, TaH, KP923879532 * TaI);
+		    TcV = T1f - T1q;
+		    TcY = TcW - TcX;
+		    TcZ = TcV + TcY;
+		    TeF = TcV - TcY;
+	       }
+	  }
+	  {
+	       E T1y, T6M, T1D, T6N, T1E, Td2, T1J, T74, T1O, T75, T1P, Td3, T21, Td8, T6W;
+	       E T6Z, T2c, Td9, T6R, T6U;
+	       {
+		    E T1v, T1x, T1u, T1w;
+		    T1v = ri[WS(ios, 2)];
+		    T1x = ii[WS(ios, 2)];
+		    T1u = W[2];
+		    T1w = W[3];
+		    T1y = FMA(T1u, T1v, T1w * T1x);
+		    T6M = FNMS(T1w, T1v, T1u * T1x);
+	       }
+	       {
+		    E T1A, T1C, T1z, T1B;
+		    T1A = ri[WS(ios, 34)];
+		    T1C = ii[WS(ios, 34)];
+		    T1z = W[66];
+		    T1B = W[67];
+		    T1D = FMA(T1z, T1A, T1B * T1C);
+		    T6N = FNMS(T1B, T1A, T1z * T1C);
+	       }
+	       T1E = T1y + T1D;
+	       Td2 = T6M + T6N;
+	       {
+		    E T1G, T1I, T1F, T1H;
+		    T1G = ri[WS(ios, 18)];
+		    T1I = ii[WS(ios, 18)];
+		    T1F = W[34];
+		    T1H = W[35];
+		    T1J = FMA(T1F, T1G, T1H * T1I);
+		    T74 = FNMS(T1H, T1G, T1F * T1I);
+	       }
+	       {
+		    E T1L, T1N, T1K, T1M;
+		    T1L = ri[WS(ios, 50)];
+		    T1N = ii[WS(ios, 50)];
+		    T1K = W[98];
+		    T1M = W[99];
+		    T1O = FMA(T1K, T1L, T1M * T1N);
+		    T75 = FNMS(T1M, T1L, T1K * T1N);
+	       }
+	       T1P = T1J + T1O;
+	       Td3 = T74 + T75;
+	       {
+		    E T1V, T6X, T20, T6Y;
+		    {
+			 E T1S, T1U, T1R, T1T;
+			 T1S = ri[WS(ios, 10)];
+			 T1U = ii[WS(ios, 10)];
+			 T1R = W[18];
+			 T1T = W[19];
+			 T1V = FMA(T1R, T1S, T1T * T1U);
+			 T6X = FNMS(T1T, T1S, T1R * T1U);
+		    }
+		    {
+			 E T1X, T1Z, T1W, T1Y;
+			 T1X = ri[WS(ios, 42)];
+			 T1Z = ii[WS(ios, 42)];
+			 T1W = W[82];
+			 T1Y = W[83];
+			 T20 = FMA(T1W, T1X, T1Y * T1Z);
+			 T6Y = FNMS(T1Y, T1X, T1W * T1Z);
+		    }
+		    T21 = T1V + T20;
+		    Td8 = T6X + T6Y;
+		    T6W = T1V - T20;
+		    T6Z = T6X - T6Y;
+	       }
+	       {
+		    E T26, T6S, T2b, T6T;
+		    {
+			 E T23, T25, T22, T24;
+			 T23 = ri[WS(ios, 58)];
+			 T25 = ii[WS(ios, 58)];
+			 T22 = W[114];
+			 T24 = W[115];
+			 T26 = FMA(T22, T23, T24 * T25);
+			 T6S = FNMS(T24, T23, T22 * T25);
+		    }
+		    {
+			 E T28, T2a, T27, T29;
+			 T28 = ri[WS(ios, 26)];
+			 T2a = ii[WS(ios, 26)];
+			 T27 = W[50];
+			 T29 = W[51];
+			 T2b = FMA(T27, T28, T29 * T2a);
+			 T6T = FNMS(T29, T28, T27 * T2a);
+		    }
+		    T2c = T26 + T2b;
+		    Td9 = T6S + T6T;
+		    T6R = T26 - T2b;
+		    T6U = T6S - T6T;
+	       }
+	       T1Q = T1E + T1P;
+	       T2d = T21 + T2c;
+	       Tfx = T1Q - T2d;
+	       Tfu = Td2 + Td3;
+	       Tfv = Td8 + Td9;
+	       Tfw = Tfu - Tfv;
+	       {
+		    E T6O, T6P, Td7, Tda;
+		    T6O = T6M - T6N;
+		    T6P = T1J - T1O;
+		    T6Q = T6O + T6P;
+		    TaM = T6O - T6P;
+		    Td7 = T1E - T1P;
+		    Tda = Td8 - Td9;
+		    Tdb = Td7 - Tda;
+		    TeJ = Td7 + Tda;
+	       }
+	       {
+		    E T6V, T70, T78, T79;
+		    T6V = T6R - T6U;
+		    T70 = T6W + T6Z;
+		    T71 = KP707106781 * (T6V - T70);
+		    TaQ = KP707106781 * (T70 + T6V);
+		    T78 = T6Z - T6W;
+		    T79 = T6R + T6U;
+		    T7a = KP707106781 * (T78 - T79);
+		    TaN = KP707106781 * (T78 + T79);
+	       }
+	       {
+		    E Td4, Td5, T73, T76;
+		    Td4 = Td2 - Td3;
+		    Td5 = T2c - T21;
+		    Td6 = Td4 - Td5;
+		    TeI = Td4 + Td5;
+		    T73 = T1y - T1D;
+		    T76 = T74 - T75;
+		    T77 = T73 - T76;
+		    TaP = T73 + T76;
+	       }
+	  }
+	  {
+	       E T2j, T7d, T2o, T7e, T2p, Tdd, T2u, T7v, T2z, T7w, T2A, Tde, T2M, Tdj, T7n;
+	       E T7q, T2X, Tdk, T7i, T7l;
+	       {
+		    E T2g, T2i, T2f, T2h;
+		    T2g = ri[WS(ios, 62)];
+		    T2i = ii[WS(ios, 62)];
+		    T2f = W[122];
+		    T2h = W[123];
+		    T2j = FMA(T2f, T2g, T2h * T2i);
+		    T7d = FNMS(T2h, T2g, T2f * T2i);
+	       }
+	       {
+		    E T2l, T2n, T2k, T2m;
+		    T2l = ri[WS(ios, 30)];
+		    T2n = ii[WS(ios, 30)];
+		    T2k = W[58];
+		    T2m = W[59];
+		    T2o = FMA(T2k, T2l, T2m * T2n);
+		    T7e = FNMS(T2m, T2l, T2k * T2n);
+	       }
+	       T2p = T2j + T2o;
+	       Tdd = T7d + T7e;
+	       {
+		    E T2r, T2t, T2q, T2s;
+		    T2r = ri[WS(ios, 14)];
+		    T2t = ii[WS(ios, 14)];
+		    T2q = W[26];
+		    T2s = W[27];
+		    T2u = FMA(T2q, T2r, T2s * T2t);
+		    T7v = FNMS(T2s, T2r, T2q * T2t);
+	       }
+	       {
+		    E T2w, T2y, T2v, T2x;
+		    T2w = ri[WS(ios, 46)];
+		    T2y = ii[WS(ios, 46)];
+		    T2v = W[90];
+		    T2x = W[91];
+		    T2z = FMA(T2v, T2w, T2x * T2y);
+		    T7w = FNMS(T2x, T2w, T2v * T2y);
+	       }
+	       T2A = T2u + T2z;
+	       Tde = T7v + T7w;
+	       {
+		    E T2G, T7o, T2L, T7p;
+		    {
+			 E T2D, T2F, T2C, T2E;
+			 T2D = ri[WS(ios, 6)];
+			 T2F = ii[WS(ios, 6)];
+			 T2C = W[10];
+			 T2E = W[11];
+			 T2G = FMA(T2C, T2D, T2E * T2F);
+			 T7o = FNMS(T2E, T2D, T2C * T2F);
+		    }
+		    {
+			 E T2I, T2K, T2H, T2J;
+			 T2I = ri[WS(ios, 38)];
+			 T2K = ii[WS(ios, 38)];
+			 T2H = W[74];
+			 T2J = W[75];
+			 T2L = FMA(T2H, T2I, T2J * T2K);
+			 T7p = FNMS(T2J, T2I, T2H * T2K);
+		    }
+		    T2M = T2G + T2L;
+		    Tdj = T7o + T7p;
+		    T7n = T2G - T2L;
+		    T7q = T7o - T7p;
+	       }
+	       {
+		    E T2R, T7j, T2W, T7k;
+		    {
+			 E T2O, T2Q, T2N, T2P;
+			 T2O = ri[WS(ios, 54)];
+			 T2Q = ii[WS(ios, 54)];
+			 T2N = W[106];
+			 T2P = W[107];
+			 T2R = FMA(T2N, T2O, T2P * T2Q);
+			 T7j = FNMS(T2P, T2O, T2N * T2Q);
+		    }
+		    {
+			 E T2T, T2V, T2S, T2U;
+			 T2T = ri[WS(ios, 22)];
+			 T2V = ii[WS(ios, 22)];
+			 T2S = W[42];
+			 T2U = W[43];
+			 T2W = FMA(T2S, T2T, T2U * T2V);
+			 T7k = FNMS(T2U, T2T, T2S * T2V);
+		    }
+		    T2X = T2R + T2W;
+		    Tdk = T7j + T7k;
+		    T7i = T2R - T2W;
+		    T7l = T7j - T7k;
+	       }
+	       T2B = T2p + T2A;
+	       T2Y = T2M + T2X;
+	       Tfz = T2B - T2Y;
+	       TfA = Tdd + Tde;
+	       TfB = Tdj + Tdk;
+	       TfC = TfA - TfB;
+	       {
+		    E T7f, T7g, Tdi, Tdl;
+		    T7f = T7d - T7e;
+		    T7g = T2u - T2z;
+		    T7h = T7f + T7g;
+		    TaW = T7f - T7g;
+		    Tdi = T2p - T2A;
+		    Tdl = Tdj - Tdk;
+		    Tdm = Tdi - Tdl;
+		    TeM = Tdi + Tdl;
+	       }
+	       {
+		    E T7m, T7r, T7z, T7A;
+		    T7m = T7i - T7l;
+		    T7r = T7n + T7q;
+		    T7s = KP707106781 * (T7m - T7r);
+		    TaU = KP707106781 * (T7r + T7m);
+		    T7z = T7q - T7n;
+		    T7A = T7i + T7l;
+		    T7B = KP707106781 * (T7z - T7A);
+		    TaX = KP707106781 * (T7z + T7A);
+	       }
+	       {
+		    E Tdf, Tdg, T7u, T7x;
+		    Tdf = Tdd - Tde;
+		    Tdg = T2X - T2M;
+		    Tdh = Tdf - Tdg;
+		    TeL = Tdf + Tdg;
+		    T7u = T2j - T2o;
+		    T7x = T7v - T7w;
+		    T7y = T7u - T7x;
+		    TaT = T7u + T7x;
+	       }
+	  }
+	  {
+	       E T4D, T9e, T4I, T9f, T4J, Te8, T4O, T8A, T4T, T8B, T4U, Te9, T56, TdS, T8G;
+	       E T8H, T5h, TdT, T8J, T8M;
+	       {
+		    E T4A, T4C, T4z, T4B;
+		    T4A = ri[WS(ios, 63)];
+		    T4C = ii[WS(ios, 63)];
+		    T4z = W[124];
+		    T4B = W[125];
+		    T4D = FMA(T4z, T4A, T4B * T4C);
+		    T9e = FNMS(T4B, T4A, T4z * T4C);
+	       }
+	       {
+		    E T4F, T4H, T4E, T4G;
+		    T4F = ri[WS(ios, 31)];
+		    T4H = ii[WS(ios, 31)];
+		    T4E = W[60];
+		    T4G = W[61];
+		    T4I = FMA(T4E, T4F, T4G * T4H);
+		    T9f = FNMS(T4G, T4F, T4E * T4H);
+	       }
+	       T4J = T4D + T4I;
+	       Te8 = T9e + T9f;
+	       {
+		    E T4L, T4N, T4K, T4M;
+		    T4L = ri[WS(ios, 15)];
+		    T4N = ii[WS(ios, 15)];
+		    T4K = W[28];
+		    T4M = W[29];
+		    T4O = FMA(T4K, T4L, T4M * T4N);
+		    T8A = FNMS(T4M, T4L, T4K * T4N);
+	       }
+	       {
+		    E T4Q, T4S, T4P, T4R;
+		    T4Q = ri[WS(ios, 47)];
+		    T4S = ii[WS(ios, 47)];
+		    T4P = W[92];
+		    T4R = W[93];
+		    T4T = FMA(T4P, T4Q, T4R * T4S);
+		    T8B = FNMS(T4R, T4Q, T4P * T4S);
+	       }
+	       T4U = T4O + T4T;
+	       Te9 = T8A + T8B;
+	       {
+		    E T50, T8E, T55, T8F;
+		    {
+			 E T4X, T4Z, T4W, T4Y;
+			 T4X = ri[WS(ios, 7)];
+			 T4Z = ii[WS(ios, 7)];
+			 T4W = W[12];
+			 T4Y = W[13];
+			 T50 = FMA(T4W, T4X, T4Y * T4Z);
+			 T8E = FNMS(T4Y, T4X, T4W * T4Z);
+		    }
+		    {
+			 E T52, T54, T51, T53;
+			 T52 = ri[WS(ios, 39)];
+			 T54 = ii[WS(ios, 39)];
+			 T51 = W[76];
+			 T53 = W[77];
+			 T55 = FMA(T51, T52, T53 * T54);
+			 T8F = FNMS(T53, T52, T51 * T54);
+		    }
+		    T56 = T50 + T55;
+		    TdS = T8E + T8F;
+		    T8G = T8E - T8F;
+		    T8H = T50 - T55;
+	       }
+	       {
+		    E T5b, T8K, T5g, T8L;
+		    {
+			 E T58, T5a, T57, T59;
+			 T58 = ri[WS(ios, 55)];
+			 T5a = ii[WS(ios, 55)];
+			 T57 = W[108];
+			 T59 = W[109];
+			 T5b = FMA(T57, T58, T59 * T5a);
+			 T8K = FNMS(T59, T58, T57 * T5a);
+		    }
+		    {
+			 E T5d, T5f, T5c, T5e;
+			 T5d = ri[WS(ios, 23)];
+			 T5f = ii[WS(ios, 23)];
+			 T5c = W[44];
+			 T5e = W[45];
+			 T5g = FMA(T5c, T5d, T5e * T5f);
+			 T8L = FNMS(T5e, T5d, T5c * T5f);
+		    }
+		    T5h = T5b + T5g;
+		    TdT = T8K + T8L;
+		    T8J = T5b - T5g;
+		    T8M = T8K - T8L;
+	       }
+	       {
+		    E T4V, T5i, Tea, Teb;
+		    T4V = T4J + T4U;
+		    T5i = T56 + T5h;
+		    T5j = T4V + T5i;
+		    TfR = T4V - T5i;
+		    Tea = Te8 - Te9;
+		    Teb = T5h - T56;
+		    Tec = Tea - Teb;
+		    Tf0 = Tea + Teb;
+	       }
+	       {
+		    E TfW, TfX, T8z, T8C;
+		    TfW = Te8 + Te9;
+		    TfX = TdS + TdT;
+		    TfY = TfW - TfX;
+		    Tgy = TfW + TfX;
+		    T8z = T4D - T4I;
+		    T8C = T8A - T8B;
+		    T8D = T8z - T8C;
+		    Tbl = T8z + T8C;
+	       }
+	       {
+		    E T8I, T8N, T9j, T9k;
+		    T8I = T8G - T8H;
+		    T8N = T8J + T8M;
+		    T8O = KP707106781 * (T8I - T8N);
+		    Tbx = KP707106781 * (T8I + T8N);
+		    T9j = T8J - T8M;
+		    T9k = T8H + T8G;
+		    T9l = KP707106781 * (T9j - T9k);
+		    Tbm = KP707106781 * (T9k + T9j);
+	       }
+	       {
+		    E TdR, TdU, T9g, T9h;
+		    TdR = T4J - T4U;
+		    TdU = TdS - TdT;
+		    TdV = TdR - TdU;
+		    TeX = TdR + TdU;
+		    T9g = T9e - T9f;
+		    T9h = T4O - T4T;
+		    T9i = T9g + T9h;
+		    Tbw = T9g - T9h;
+	       }
+	  }
+	  {
+	       E T36, T7G, T3b, T7H, T3c, Tdq, T3h, T8m, T3m, T8n, T3n, Tdr, T3z, TdI, T7Q;
+	       E T7T, T3K, TdJ, T7L, T7O;
+	       {
+		    E T33, T35, T32, T34;
+		    T33 = ri[WS(ios, 1)];
+		    T35 = ii[WS(ios, 1)];
+		    T32 = W[0];
+		    T34 = W[1];
+		    T36 = FMA(T32, T33, T34 * T35);
+		    T7G = FNMS(T34, T33, T32 * T35);
+	       }
+	       {
+		    E T38, T3a, T37, T39;
+		    T38 = ri[WS(ios, 33)];
+		    T3a = ii[WS(ios, 33)];
+		    T37 = W[64];
+		    T39 = W[65];
+		    T3b = FMA(T37, T38, T39 * T3a);
+		    T7H = FNMS(T39, T38, T37 * T3a);
+	       }
+	       T3c = T36 + T3b;
+	       Tdq = T7G + T7H;
+	       {
+		    E T3e, T3g, T3d, T3f;
+		    T3e = ri[WS(ios, 17)];
+		    T3g = ii[WS(ios, 17)];
+		    T3d = W[32];
+		    T3f = W[33];
+		    T3h = FMA(T3d, T3e, T3f * T3g);
+		    T8m = FNMS(T3f, T3e, T3d * T3g);
+	       }
+	       {
+		    E T3j, T3l, T3i, T3k;
+		    T3j = ri[WS(ios, 49)];
+		    T3l = ii[WS(ios, 49)];
+		    T3i = W[96];
+		    T3k = W[97];
+		    T3m = FMA(T3i, T3j, T3k * T3l);
+		    T8n = FNMS(T3k, T3j, T3i * T3l);
+	       }
+	       T3n = T3h + T3m;
+	       Tdr = T8m + T8n;
+	       {
+		    E T3t, T7R, T3y, T7S;
+		    {
+			 E T3q, T3s, T3p, T3r;
+			 T3q = ri[WS(ios, 9)];
+			 T3s = ii[WS(ios, 9)];
+			 T3p = W[16];
+			 T3r = W[17];
+			 T3t = FMA(T3p, T3q, T3r * T3s);
+			 T7R = FNMS(T3r, T3q, T3p * T3s);
+		    }
+		    {
+			 E T3v, T3x, T3u, T3w;
+			 T3v = ri[WS(ios, 41)];
+			 T3x = ii[WS(ios, 41)];
+			 T3u = W[80];
+			 T3w = W[81];
+			 T3y = FMA(T3u, T3v, T3w * T3x);
+			 T7S = FNMS(T3w, T3v, T3u * T3x);
+		    }
+		    T3z = T3t + T3y;
+		    TdI = T7R + T7S;
+		    T7Q = T3t - T3y;
+		    T7T = T7R - T7S;
+	       }
+	       {
+		    E T3E, T7M, T3J, T7N;
+		    {
+			 E T3B, T3D, T3A, T3C;
+			 T3B = ri[WS(ios, 57)];
+			 T3D = ii[WS(ios, 57)];
+			 T3A = W[112];
+			 T3C = W[113];
+			 T3E = FMA(T3A, T3B, T3C * T3D);
+			 T7M = FNMS(T3C, T3B, T3A * T3D);
+		    }
+		    {
+			 E T3G, T3I, T3F, T3H;
+			 T3G = ri[WS(ios, 25)];
+			 T3I = ii[WS(ios, 25)];
+			 T3F = W[48];
+			 T3H = W[49];
+			 T3J = FMA(T3F, T3G, T3H * T3I);
+			 T7N = FNMS(T3H, T3G, T3F * T3I);
+		    }
+		    T3K = T3E + T3J;
+		    TdJ = T7M + T7N;
+		    T7L = T3E - T3J;
+		    T7O = T7M - T7N;
+	       }
+	       {
+		    E T3o, T3L, TdH, TdK;
+		    T3o = T3c + T3n;
+		    T3L = T3z + T3K;
+		    T3M = T3o + T3L;
+		    TfL = T3o - T3L;
+		    TdH = T3c - T3n;
+		    TdK = TdI - TdJ;
+		    TdL = TdH - TdK;
+		    TeQ = TdH + TdK;
+	       }
+	       {
+		    E TfG, TfH, T7I, T7J;
+		    TfG = Tdq + Tdr;
+		    TfH = TdI + TdJ;
+		    TfI = TfG - TfH;
+		    Tgt = TfG + TfH;
+		    T7I = T7G - T7H;
+		    T7J = T3h - T3m;
+		    T7K = T7I + T7J;
+		    Tb2 = T7I - T7J;
+	       }
+	       {
+		    E T7P, T7U, T8q, T8r;
+		    T7P = T7L - T7O;
+		    T7U = T7Q + T7T;
+		    T7V = KP707106781 * (T7P - T7U);
+		    Tbe = KP707106781 * (T7U + T7P);
+		    T8q = T7T - T7Q;
+		    T8r = T7L + T7O;
+		    T8s = KP707106781 * (T8q - T8r);
+		    Tb3 = KP707106781 * (T8q + T8r);
+	       }
+	       {
+		    E Tds, Tdt, T8l, T8o;
+		    Tds = Tdq - Tdr;
+		    Tdt = T3K - T3z;
+		    Tdu = Tds - Tdt;
+		    TeT = Tds + Tdt;
+		    T8l = T36 - T3b;
+		    T8o = T8m - T8n;
+		    T8p = T8l - T8o;
+		    Tbd = T8l + T8o;
+	       }
+	  }
+	  {
+	       E T3X, TdB, T8a, T8d, T4v, Tdx, T80, T85, T48, TdC, T8b, T8g, T4k, Tdw, T7X;
+	       E T84;
+	       {
+		    E T3R, T88, T3W, T89;
+		    {
+			 E T3O, T3Q, T3N, T3P;
+			 T3O = ri[WS(ios, 5)];
+			 T3Q = ii[WS(ios, 5)];
+			 T3N = W[8];
+			 T3P = W[9];
+			 T3R = FMA(T3N, T3O, T3P * T3Q);
+			 T88 = FNMS(T3P, T3O, T3N * T3Q);
+		    }
+		    {
+			 E T3T, T3V, T3S, T3U;
+			 T3T = ri[WS(ios, 37)];
+			 T3V = ii[WS(ios, 37)];
+			 T3S = W[72];
+			 T3U = W[73];
+			 T3W = FMA(T3S, T3T, T3U * T3V);
+			 T89 = FNMS(T3U, T3T, T3S * T3V);
+		    }
+		    T3X = T3R + T3W;
+		    TdB = T88 + T89;
+		    T8a = T88 - T89;
+		    T8d = T3R - T3W;
+	       }
+	       {
+		    E T4p, T7Y, T4u, T7Z;
+		    {
+			 E T4m, T4o, T4l, T4n;
+			 T4m = ri[WS(ios, 13)];
+			 T4o = ii[WS(ios, 13)];
+			 T4l = W[24];
+			 T4n = W[25];
+			 T4p = FMA(T4l, T4m, T4n * T4o);
+			 T7Y = FNMS(T4n, T4m, T4l * T4o);
+		    }
+		    {
+			 E T4r, T4t, T4q, T4s;
+			 T4r = ri[WS(ios, 45)];
+			 T4t = ii[WS(ios, 45)];
+			 T4q = W[88];
+			 T4s = W[89];
+			 T4u = FMA(T4q, T4r, T4s * T4t);
+			 T7Z = FNMS(T4s, T4r, T4q * T4t);
+		    }
+		    T4v = T4p + T4u;
+		    Tdx = T7Y + T7Z;
+		    T80 = T7Y - T7Z;
+		    T85 = T4p - T4u;
+	       }
+	       {
+		    E T42, T8e, T47, T8f;
+		    {
+			 E T3Z, T41, T3Y, T40;
+			 T3Z = ri[WS(ios, 21)];
+			 T41 = ii[WS(ios, 21)];
+			 T3Y = W[40];
+			 T40 = W[41];
+			 T42 = FMA(T3Y, T3Z, T40 * T41);
+			 T8e = FNMS(T40, T3Z, T3Y * T41);
+		    }
+		    {
+			 E T44, T46, T43, T45;
+			 T44 = ri[WS(ios, 53)];
+			 T46 = ii[WS(ios, 53)];
+			 T43 = W[104];
+			 T45 = W[105];
+			 T47 = FMA(T43, T44, T45 * T46);
+			 T8f = FNMS(T45, T44, T43 * T46);
+		    }
+		    T48 = T42 + T47;
+		    TdC = T8e + T8f;
+		    T8b = T42 - T47;
+		    T8g = T8e - T8f;
+	       }
+	       {
+		    E T4e, T82, T4j, T83;
+		    {
+			 E T4b, T4d, T4a, T4c;
+			 T4b = ri[WS(ios, 61)];
+			 T4d = ii[WS(ios, 61)];
+			 T4a = W[120];
+			 T4c = W[121];
+			 T4e = FMA(T4a, T4b, T4c * T4d);
+			 T82 = FNMS(T4c, T4b, T4a * T4d);
+		    }
+		    {
+			 E T4g, T4i, T4f, T4h;
+			 T4g = ri[WS(ios, 29)];
+			 T4i = ii[WS(ios, 29)];
+			 T4f = W[56];
+			 T4h = W[57];
+			 T4j = FMA(T4f, T4g, T4h * T4i);
+			 T83 = FNMS(T4h, T4g, T4f * T4i);
+		    }
+		    T4k = T4e + T4j;
+		    Tdw = T82 + T83;
+		    T7X = T4e - T4j;
+		    T84 = T82 - T83;
+	       }
+	       {
+		    E T49, T4w, TdA, TdD;
+		    T49 = T3X + T48;
+		    T4w = T4k + T4v;
+		    T4x = T49 + T4w;
+		    TfJ = T4w - T49;
+		    TdA = T3X - T48;
+		    TdD = TdB - TdC;
+		    TdE = TdA + TdD;
+		    TdM = TdD - TdA;
+	       }
+	       {
+		    E TfM, TfN, T81, T86;
+		    TfM = TdB + TdC;
+		    TfN = Tdw + Tdx;
+		    TfO = TfM - TfN;
+		    Tgu = TfM + TfN;
+		    T81 = T7X - T80;
+		    T86 = T84 + T85;
+		    T87 = FNMS(KP923879532, T86, KP382683432 * T81);
+		    T8v = FMA(KP382683432, T86, KP923879532 * T81);
+	       }
+	       {
+		    E T8c, T8h, Tb8, Tb9;
+		    T8c = T8a + T8b;
+		    T8h = T8d - T8g;
+		    T8i = FMA(KP923879532, T8c, KP382683432 * T8h);
+		    T8u = FNMS(KP923879532, T8h, KP382683432 * T8c);
+		    Tb8 = T8a - T8b;
+		    Tb9 = T8d + T8g;
+		    Tba = FMA(KP382683432, Tb8, KP923879532 * Tb9);
+		    Tbg = FNMS(KP382683432, Tb9, KP923879532 * Tb8);
+	       }
+	       {
+		    E Tdv, Tdy, Tb5, Tb6;
+		    Tdv = T4k - T4v;
+		    Tdy = Tdw - Tdx;
+		    Tdz = Tdv - Tdy;
+		    TdN = Tdv + Tdy;
+		    Tb5 = T7X + T80;
+		    Tb6 = T84 - T85;
+		    Tb7 = FNMS(KP382683432, Tb6, KP923879532 * Tb5);
+		    Tbh = FMA(KP923879532, Tb6, KP382683432 * Tb5);
+	       }
+	  }
+	  {
+	       E T5u, TdW, T8S, T8V, T62, Te3, T94, T99, T5F, TdX, T8T, T8Y, T5R, Te2, T93;
+	       E T96;
+	       {
+		    E T5o, T8Q, T5t, T8R;
+		    {
+			 E T5l, T5n, T5k, T5m;
+			 T5l = ri[WS(ios, 3)];
+			 T5n = ii[WS(ios, 3)];
+			 T5k = W[4];
+			 T5m = W[5];
+			 T5o = FMA(T5k, T5l, T5m * T5n);
+			 T8Q = FNMS(T5m, T5l, T5k * T5n);
+		    }
+		    {
+			 E T5q, T5s, T5p, T5r;
+			 T5q = ri[WS(ios, 35)];
+			 T5s = ii[WS(ios, 35)];
+			 T5p = W[68];
+			 T5r = W[69];
+			 T5t = FMA(T5p, T5q, T5r * T5s);
+			 T8R = FNMS(T5r, T5q, T5p * T5s);
+		    }
+		    T5u = T5o + T5t;
+		    TdW = T8Q + T8R;
+		    T8S = T8Q - T8R;
+		    T8V = T5o - T5t;
+	       }
+	       {
+		    E T5W, T97, T61, T98;
+		    {
+			 E T5T, T5V, T5S, T5U;
+			 T5T = ri[WS(ios, 11)];
+			 T5V = ii[WS(ios, 11)];
+			 T5S = W[20];
+			 T5U = W[21];
+			 T5W = FMA(T5S, T5T, T5U * T5V);
+			 T97 = FNMS(T5U, T5T, T5S * T5V);
+		    }
+		    {
+			 E T5Y, T60, T5X, T5Z;
+			 T5Y = ri[WS(ios, 43)];
+			 T60 = ii[WS(ios, 43)];
+			 T5X = W[84];
+			 T5Z = W[85];
+			 T61 = FMA(T5X, T5Y, T5Z * T60);
+			 T98 = FNMS(T5Z, T5Y, T5X * T60);
+		    }
+		    T62 = T5W + T61;
+		    Te3 = T97 + T98;
+		    T94 = T5W - T61;
+		    T99 = T97 - T98;
+	       }
+	       {
+		    E T5z, T8W, T5E, T8X;
+		    {
+			 E T5w, T5y, T5v, T5x;
+			 T5w = ri[WS(ios, 19)];
+			 T5y = ii[WS(ios, 19)];
+			 T5v = W[36];
+			 T5x = W[37];
+			 T5z = FMA(T5v, T5w, T5x * T5y);
+			 T8W = FNMS(T5x, T5w, T5v * T5y);
+		    }
+		    {
+			 E T5B, T5D, T5A, T5C;
+			 T5B = ri[WS(ios, 51)];
+			 T5D = ii[WS(ios, 51)];
+			 T5A = W[100];
+			 T5C = W[101];
+			 T5E = FMA(T5A, T5B, T5C * T5D);
+			 T8X = FNMS(T5C, T5B, T5A * T5D);
+		    }
+		    T5F = T5z + T5E;
+		    TdX = T8W + T8X;
+		    T8T = T5z - T5E;
+		    T8Y = T8W - T8X;
+	       }
+	       {
+		    E T5L, T91, T5Q, T92;
+		    {
+			 E T5I, T5K, T5H, T5J;
+			 T5I = ri[WS(ios, 59)];
+			 T5K = ii[WS(ios, 59)];
+			 T5H = W[116];
+			 T5J = W[117];
+			 T5L = FMA(T5H, T5I, T5J * T5K);
+			 T91 = FNMS(T5J, T5I, T5H * T5K);
+		    }
+		    {
+			 E T5N, T5P, T5M, T5O;
+			 T5N = ri[WS(ios, 27)];
+			 T5P = ii[WS(ios, 27)];
+			 T5M = W[52];
+			 T5O = W[53];
+			 T5Q = FMA(T5M, T5N, T5O * T5P);
+			 T92 = FNMS(T5O, T5N, T5M * T5P);
+		    }
+		    T5R = T5L + T5Q;
+		    Te2 = T91 + T92;
+		    T93 = T91 - T92;
+		    T96 = T5L - T5Q;
+	       }
+	       {
+		    E T5G, T63, Te1, Te4;
+		    T5G = T5u + T5F;
+		    T63 = T5R + T62;
+		    T64 = T5G + T63;
+		    TfZ = T63 - T5G;
+		    Te1 = T5R - T62;
+		    Te4 = Te2 - Te3;
+		    Te5 = Te1 + Te4;
+		    Ted = Te1 - Te4;
+	       }
+	       {
+		    E TfS, TfT, T8U, T8Z;
+		    TfS = TdW + TdX;
+		    TfT = Te2 + Te3;
+		    TfU = TfS - TfT;
+		    Tgz = TfS + TfT;
+		    T8U = T8S + T8T;
+		    T8Z = T8V - T8Y;
+		    T90 = FNMS(KP923879532, T8Z, KP382683432 * T8U);
+		    T9o = FMA(KP923879532, T8U, KP382683432 * T8Z);
+	       }
+	       {
+		    E T95, T9a, Tbr, Tbs;
+		    T95 = T93 + T94;
+		    T9a = T96 - T99;
+		    T9b = FMA(KP382683432, T95, KP923879532 * T9a);
+		    T9n = FNMS(KP923879532, T95, KP382683432 * T9a);
+		    Tbr = T93 - T94;
+		    Tbs = T96 + T99;
+		    Tbt = FMA(KP923879532, Tbr, KP382683432 * Tbs);
+		    Tbz = FNMS(KP382683432, Tbr, KP923879532 * Tbs);
+	       }
+	       {
+		    E TdY, TdZ, Tbo, Tbp;
+		    TdY = TdW - TdX;
+		    TdZ = T5u - T5F;
+		    Te0 = TdY - TdZ;
+		    Tee = TdZ + TdY;
+		    Tbo = T8S - T8T;
+		    Tbp = T8V + T8Y;
+		    Tbq = FNMS(KP382683432, Tbp, KP923879532 * Tbo);
+		    TbA = FMA(KP382683432, Tbo, KP923879532 * Tbp);
+	       }
+	  }
+	  {
+	       E T1t, Tgn, TgK, TgL, TgV, Th1, T30, Th0, T66, TgX, Tgw, TgE, TgB, TgF, Tgq;
+	       E TgM;
+	       {
+		    E TH, T1s, TgI, TgJ;
+		    TH = Tj + TG;
+		    T1s = T14 + T1r;
+		    T1t = TH + T1s;
+		    Tgn = TH - T1s;
+		    TgI = Tgt + Tgu;
+		    TgJ = Tgy + Tgz;
+		    TgK = TgI - TgJ;
+		    TgL = TgI + TgJ;
+	       }
+	       {
+		    E TgN, TgU, T2e, T2Z;
+		    TgN = Tfq + Tfr;
+		    TgU = TgO + TgT;
+		    TgV = TgN + TgU;
+		    Th1 = TgU - TgN;
+		    T2e = T1Q + T2d;
+		    T2Z = T2B + T2Y;
+		    T30 = T2e + T2Z;
+		    Th0 = T2Z - T2e;
+	       }
+	       {
+		    E T4y, T65, Tgs, Tgv;
+		    T4y = T3M + T4x;
+		    T65 = T5j + T64;
+		    T66 = T4y + T65;
+		    TgX = T65 - T4y;
+		    Tgs = T3M - T4x;
+		    Tgv = Tgt - Tgu;
+		    Tgw = Tgs + Tgv;
+		    TgE = Tgv - Tgs;
+	       }
+	       {
+		    E Tgx, TgA, Tgo, Tgp;
+		    Tgx = T5j - T64;
+		    TgA = Tgy - Tgz;
+		    TgB = Tgx - TgA;
+		    TgF = Tgx + TgA;
+		    Tgo = Tfu + Tfv;
+		    Tgp = TfA + TfB;
+		    Tgq = Tgo - Tgp;
+		    TgM = Tgo + Tgp;
+	       }
+	       {
+		    E T31, TgW, TgH, TgY;
+		    T31 = T1t + T30;
+		    ri[WS(ios, 32)] = T31 - T66;
+		    ri[0] = T31 + T66;
+		    TgW = TgM + TgV;
+		    ii[0] = TgL + TgW;
+		    ii[WS(ios, 32)] = TgW - TgL;
+		    TgH = T1t - T30;
+		    ri[WS(ios, 48)] = TgH - TgK;
+		    ri[WS(ios, 16)] = TgH + TgK;
+		    TgY = TgV - TgM;
+		    ii[WS(ios, 16)] = TgX + TgY;
+		    ii[WS(ios, 48)] = TgY - TgX;
+	       }
+	       {
+		    E Tgr, TgC, TgZ, Th2;
+		    Tgr = Tgn + Tgq;
+		    TgC = KP707106781 * (Tgw + TgB);
+		    ri[WS(ios, 40)] = Tgr - TgC;
+		    ri[WS(ios, 8)] = Tgr + TgC;
+		    TgZ = KP707106781 * (TgE + TgF);
+		    Th2 = Th0 + Th1;
+		    ii[WS(ios, 8)] = TgZ + Th2;
+		    ii[WS(ios, 40)] = Th2 - TgZ;
+	       }
+	       {
+		    E TgD, TgG, Th3, Th4;
+		    TgD = Tgn - Tgq;
+		    TgG = KP707106781 * (TgE - TgF);
+		    ri[WS(ios, 56)] = TgD - TgG;
+		    ri[WS(ios, 24)] = TgD + TgG;
+		    Th3 = KP707106781 * (TgB - Tgw);
+		    Th4 = Th1 - Th0;
+		    ii[WS(ios, 24)] = Th3 + Th4;
+		    ii[WS(ios, 56)] = Th4 - Th3;
+	       }
+	  }
+	  {
+	       E Tft, Tg7, Tgh, Tgl, Th9, Thf, TfE, Th6, TfQ, Tg4, Tga, The, Tge, Tgk, Tg1;
+	       E Tg5;
+	       {
+		    E Tfp, Tfs, Tgf, Tgg;
+		    Tfp = Tj - TG;
+		    Tfs = Tfq - Tfr;
+		    Tft = Tfp - Tfs;
+		    Tg7 = Tfp + Tfs;
+		    Tgf = TfR + TfU;
+		    Tgg = TfY + TfZ;
+		    Tgh = FNMS(KP382683432, Tgg, KP923879532 * Tgf);
+		    Tgl = FMA(KP923879532, Tgg, KP382683432 * Tgf);
+	       }
+	       {
+		    E Th7, Th8, Tfy, TfD;
+		    Th7 = T1r - T14;
+		    Th8 = TgT - TgO;
+		    Th9 = Th7 + Th8;
+		    Thf = Th8 - Th7;
+		    Tfy = Tfw - Tfx;
+		    TfD = Tfz + TfC;
+		    TfE = KP707106781 * (Tfy - TfD);
+		    Th6 = KP707106781 * (Tfy + TfD);
+	       }
+	       {
+		    E TfK, TfP, Tg8, Tg9;
+		    TfK = TfI - TfJ;
+		    TfP = TfL - TfO;
+		    TfQ = FMA(KP923879532, TfK, KP382683432 * TfP);
+		    Tg4 = FNMS(KP923879532, TfP, KP382683432 * TfK);
+		    Tg8 = Tfx + Tfw;
+		    Tg9 = Tfz - TfC;
+		    Tga = KP707106781 * (Tg8 + Tg9);
+		    The = KP707106781 * (Tg9 - Tg8);
+	       }
+	       {
+		    E Tgc, Tgd, TfV, Tg0;
+		    Tgc = TfI + TfJ;
+		    Tgd = TfL + TfO;
+		    Tge = FMA(KP382683432, Tgc, KP923879532 * Tgd);
+		    Tgk = FNMS(KP382683432, Tgd, KP923879532 * Tgc);
+		    TfV = TfR - TfU;
+		    Tg0 = TfY - TfZ;
+		    Tg1 = FNMS(KP923879532, Tg0, KP382683432 * TfV);
+		    Tg5 = FMA(KP382683432, Tg0, KP923879532 * TfV);
+	       }
+	       {
+		    E TfF, Tg2, Thd, Thg;
+		    TfF = Tft + TfE;
+		    Tg2 = TfQ + Tg1;
+		    ri[WS(ios, 44)] = TfF - Tg2;
+		    ri[WS(ios, 12)] = TfF + Tg2;
+		    Thd = Tg4 + Tg5;
+		    Thg = The + Thf;
+		    ii[WS(ios, 12)] = Thd + Thg;
+		    ii[WS(ios, 44)] = Thg - Thd;
+	       }
+	       {
+		    E Tg3, Tg6, Thh, Thi;
+		    Tg3 = Tft - TfE;
+		    Tg6 = Tg4 - Tg5;
+		    ri[WS(ios, 60)] = Tg3 - Tg6;
+		    ri[WS(ios, 28)] = Tg3 + Tg6;
+		    Thh = Tg1 - TfQ;
+		    Thi = Thf - The;
+		    ii[WS(ios, 28)] = Thh + Thi;
+		    ii[WS(ios, 60)] = Thi - Thh;
+	       }
+	       {
+		    E Tgb, Tgi, Th5, Tha;
+		    Tgb = Tg7 + Tga;
+		    Tgi = Tge + Tgh;
+		    ri[WS(ios, 36)] = Tgb - Tgi;
+		    ri[WS(ios, 4)] = Tgb + Tgi;
+		    Th5 = Tgk + Tgl;
+		    Tha = Th6 + Th9;
+		    ii[WS(ios, 4)] = Th5 + Tha;
+		    ii[WS(ios, 36)] = Tha - Th5;
+	       }
+	       {
+		    E Tgj, Tgm, Thb, Thc;
+		    Tgj = Tg7 - Tga;
+		    Tgm = Tgk - Tgl;
+		    ri[WS(ios, 52)] = Tgj - Tgm;
+		    ri[WS(ios, 20)] = Tgj + Tgm;
+		    Thb = Tgh - Tge;
+		    Thc = Th9 - Th6;
+		    ii[WS(ios, 20)] = Thb + Thc;
+		    ii[WS(ios, 52)] = Thc - Thb;
+	       }
+	  }
+	  {
+	       E Td1, Ten, Tdo, ThA, ThD, ThJ, Teq, ThI, Teh, TeB, Tel, Tex, TdQ, TeA, Tek;
+	       E Teu;
+	       {
+		    E TcP, Td0, Teo, Tep;
+		    TcP = TcL - TcO;
+		    Td0 = KP707106781 * (TcU - TcZ);
+		    Td1 = TcP - Td0;
+		    Ten = TcP + Td0;
+		    {
+			 E Tdc, Tdn, ThB, ThC;
+			 Tdc = FNMS(KP923879532, Tdb, KP382683432 * Td6);
+			 Tdn = FMA(KP382683432, Tdh, KP923879532 * Tdm);
+			 Tdo = Tdc - Tdn;
+			 ThA = Tdc + Tdn;
+			 ThB = KP707106781 * (TeF - TeE);
+			 ThC = Thn - Thm;
+			 ThD = ThB + ThC;
+			 ThJ = ThC - ThB;
+		    }
+		    Teo = FMA(KP923879532, Td6, KP382683432 * Tdb);
+		    Tep = FNMS(KP923879532, Tdh, KP382683432 * Tdm);
+		    Teq = Teo + Tep;
+		    ThI = Tep - Teo;
+		    {
+			 E Te7, Tev, Teg, Tew, Te6, Tef;
+			 Te6 = KP707106781 * (Te0 - Te5);
+			 Te7 = TdV - Te6;
+			 Tev = TdV + Te6;
+			 Tef = KP707106781 * (Ted - Tee);
+			 Teg = Tec - Tef;
+			 Tew = Tec + Tef;
+			 Teh = FNMS(KP980785280, Teg, KP195090322 * Te7);
+			 TeB = FMA(KP831469612, Tew, KP555570233 * Tev);
+			 Tel = FMA(KP195090322, Teg, KP980785280 * Te7);
+			 Tex = FNMS(KP555570233, Tew, KP831469612 * Tev);
+		    }
+		    {
+			 E TdG, Tes, TdP, Tet, TdF, TdO;
+			 TdF = KP707106781 * (Tdz - TdE);
+			 TdG = Tdu - TdF;
+			 Tes = Tdu + TdF;
+			 TdO = KP707106781 * (TdM - TdN);
+			 TdP = TdL - TdO;
+			 Tet = TdL + TdO;
+			 TdQ = FMA(KP980785280, TdG, KP195090322 * TdP);
+			 TeA = FNMS(KP555570233, Tet, KP831469612 * Tes);
+			 Tek = FNMS(KP980785280, TdP, KP195090322 * TdG);
+			 Teu = FMA(KP555570233, Tes, KP831469612 * Tet);
+		    }
+	       }
+	       {
+		    E Tdp, Tei, ThH, ThK;
+		    Tdp = Td1 + Tdo;
+		    Tei = TdQ + Teh;
+		    ri[WS(ios, 46)] = Tdp - Tei;
+		    ri[WS(ios, 14)] = Tdp + Tei;
+		    ThH = Tek + Tel;
+		    ThK = ThI + ThJ;
+		    ii[WS(ios, 14)] = ThH + ThK;
+		    ii[WS(ios, 46)] = ThK - ThH;
+	       }
+	       {
+		    E Tej, Tem, ThL, ThM;
+		    Tej = Td1 - Tdo;
+		    Tem = Tek - Tel;
+		    ri[WS(ios, 62)] = Tej - Tem;
+		    ri[WS(ios, 30)] = Tej + Tem;
+		    ThL = Teh - TdQ;
+		    ThM = ThJ - ThI;
+		    ii[WS(ios, 30)] = ThL + ThM;
+		    ii[WS(ios, 62)] = ThM - ThL;
+	       }
+	       {
+		    E Ter, Tey, Thz, ThE;
+		    Ter = Ten + Teq;
+		    Tey = Teu + Tex;
+		    ri[WS(ios, 38)] = Ter - Tey;
+		    ri[WS(ios, 6)] = Ter + Tey;
+		    Thz = TeA + TeB;
+		    ThE = ThA + ThD;
+		    ii[WS(ios, 6)] = Thz + ThE;
+		    ii[WS(ios, 38)] = ThE - Thz;
+	       }
+	       {
+		    E Tez, TeC, ThF, ThG;
+		    Tez = Ten - Teq;
+		    TeC = TeA - TeB;
+		    ri[WS(ios, 54)] = Tez - TeC;
+		    ri[WS(ios, 22)] = Tez + TeC;
+		    ThF = Tex - Teu;
+		    ThG = ThD - ThA;
+		    ii[WS(ios, 22)] = ThF + ThG;
+		    ii[WS(ios, 54)] = ThG - ThF;
+	       }
+	  }
+	  {
+	       E TeH, Tf9, TeO, Thk, Thp, Thv, Tfc, Thu, Tf3, Tfn, Tf7, Tfj, TeW, Tfm, Tf6;
+	       E Tfg;
+	       {
+		    E TeD, TeG, Tfa, Tfb;
+		    TeD = TcL + TcO;
+		    TeG = KP707106781 * (TeE + TeF);
+		    TeH = TeD - TeG;
+		    Tf9 = TeD + TeG;
+		    {
+			 E TeK, TeN, Thl, Tho;
+			 TeK = FNMS(KP382683432, TeJ, KP923879532 * TeI);
+			 TeN = FMA(KP923879532, TeL, KP382683432 * TeM);
+			 TeO = TeK - TeN;
+			 Thk = TeK + TeN;
+			 Thl = KP707106781 * (TcU + TcZ);
+			 Tho = Thm + Thn;
+			 Thp = Thl + Tho;
+			 Thv = Tho - Thl;
+		    }
+		    Tfa = FMA(KP382683432, TeI, KP923879532 * TeJ);
+		    Tfb = FNMS(KP382683432, TeL, KP923879532 * TeM);
+		    Tfc = Tfa + Tfb;
+		    Thu = Tfb - Tfa;
+		    {
+			 E TeZ, Tfh, Tf2, Tfi, TeY, Tf1;
+			 TeY = KP707106781 * (Tee + Ted);
+			 TeZ = TeX - TeY;
+			 Tfh = TeX + TeY;
+			 Tf1 = KP707106781 * (Te0 + Te5);
+			 Tf2 = Tf0 - Tf1;
+			 Tfi = Tf0 + Tf1;
+			 Tf3 = FNMS(KP831469612, Tf2, KP555570233 * TeZ);
+			 Tfn = FMA(KP195090322, Tfh, KP980785280 * Tfi);
+			 Tf7 = FMA(KP831469612, TeZ, KP555570233 * Tf2);
+			 Tfj = FNMS(KP195090322, Tfi, KP980785280 * Tfh);
+		    }
+		    {
+			 E TeS, Tfe, TeV, Tff, TeR, TeU;
+			 TeR = KP707106781 * (TdE + Tdz);
+			 TeS = TeQ - TeR;
+			 Tfe = TeQ + TeR;
+			 TeU = KP707106781 * (TdM + TdN);
+			 TeV = TeT - TeU;
+			 Tff = TeT + TeU;
+			 TeW = FMA(KP555570233, TeS, KP831469612 * TeV);
+			 Tfm = FNMS(KP195090322, Tfe, KP980785280 * Tff);
+			 Tf6 = FNMS(KP831469612, TeS, KP555570233 * TeV);
+			 Tfg = FMA(KP980785280, Tfe, KP195090322 * Tff);
+		    }
+	       }
+	       {
+		    E TeP, Tf4, Tht, Thw;
+		    TeP = TeH + TeO;
+		    Tf4 = TeW + Tf3;
+		    ri[WS(ios, 42)] = TeP - Tf4;
+		    ri[WS(ios, 10)] = TeP + Tf4;
+		    Tht = Tf6 + Tf7;
+		    Thw = Thu + Thv;
+		    ii[WS(ios, 10)] = Tht + Thw;
+		    ii[WS(ios, 42)] = Thw - Tht;
+	       }
+	       {
+		    E Tf5, Tf8, Thx, Thy;
+		    Tf5 = TeH - TeO;
+		    Tf8 = Tf6 - Tf7;
+		    ri[WS(ios, 58)] = Tf5 - Tf8;
+		    ri[WS(ios, 26)] = Tf5 + Tf8;
+		    Thx = Tf3 - TeW;
+		    Thy = Thv - Thu;
+		    ii[WS(ios, 26)] = Thx + Thy;
+		    ii[WS(ios, 58)] = Thy - Thx;
+	       }
+	       {
+		    E Tfd, Tfk, Thj, Thq;
+		    Tfd = Tf9 + Tfc;
+		    Tfk = Tfg + Tfj;
+		    ri[WS(ios, 34)] = Tfd - Tfk;
+		    ri[WS(ios, 2)] = Tfd + Tfk;
+		    Thj = Tfm + Tfn;
+		    Thq = Thk + Thp;
+		    ii[WS(ios, 2)] = Thj + Thq;
+		    ii[WS(ios, 34)] = Thq - Thj;
+	       }
+	       {
+		    E Tfl, Tfo, Thr, Ths;
+		    Tfl = Tf9 - Tfc;
+		    Tfo = Tfm - Tfn;
+		    ri[WS(ios, 50)] = Tfl - Tfo;
+		    ri[WS(ios, 18)] = Tfl + Tfo;
+		    Thr = Tfj - Tfg;
+		    Ths = Thp - Thk;
+		    ii[WS(ios, 18)] = Thr + Ths;
+		    ii[WS(ios, 50)] = Ths - Thr;
+	       }
+	  }
+	  {
+	       E T6L, T9x, TiD, TiJ, T7E, TiI, T9A, TiA, T8y, T9K, T9u, T9E, T9r, T9L, T9v;
+	       E T9H;
+	       {
+		    E T6n, T6K, TiB, TiC;
+		    T6n = T6b - T6m;
+		    T6K = T6y - T6J;
+		    T6L = T6n - T6K;
+		    T9x = T6n + T6K;
+		    TiB = T9P - T9O;
+		    TiC = Tin - Tim;
+		    TiD = TiB + TiC;
+		    TiJ = TiC - TiB;
+	       }
+	       {
+		    E T7c, T9y, T7D, T9z;
+		    {
+			 E T72, T7b, T7t, T7C;
+			 T72 = T6Q - T71;
+			 T7b = T77 - T7a;
+			 T7c = FNMS(KP980785280, T7b, KP195090322 * T72);
+			 T9y = FMA(KP980785280, T72, KP195090322 * T7b);
+			 T7t = T7h - T7s;
+			 T7C = T7y - T7B;
+			 T7D = FMA(KP195090322, T7t, KP980785280 * T7C);
+			 T9z = FNMS(KP980785280, T7t, KP195090322 * T7C);
+		    }
+		    T7E = T7c - T7D;
+		    TiI = T9z - T9y;
+		    T9A = T9y + T9z;
+		    TiA = T7c + T7D;
+	       }
+	       {
+		    E T8k, T9C, T8x, T9D;
+		    {
+			 E T7W, T8j, T8t, T8w;
+			 T7W = T7K - T7V;
+			 T8j = T87 - T8i;
+			 T8k = T7W - T8j;
+			 T9C = T7W + T8j;
+			 T8t = T8p - T8s;
+			 T8w = T8u - T8v;
+			 T8x = T8t - T8w;
+			 T9D = T8t + T8w;
+		    }
+		    T8y = FMA(KP995184726, T8k, KP098017140 * T8x);
+		    T9K = FNMS(KP634393284, T9D, KP773010453 * T9C);
+		    T9u = FNMS(KP995184726, T8x, KP098017140 * T8k);
+		    T9E = FMA(KP634393284, T9C, KP773010453 * T9D);
+	       }
+	       {
+		    E T9d, T9F, T9q, T9G;
+		    {
+			 E T8P, T9c, T9m, T9p;
+			 T8P = T8D - T8O;
+			 T9c = T90 - T9b;
+			 T9d = T8P - T9c;
+			 T9F = T8P + T9c;
+			 T9m = T9i - T9l;
+			 T9p = T9n - T9o;
+			 T9q = T9m - T9p;
+			 T9G = T9m + T9p;
+		    }
+		    T9r = FNMS(KP995184726, T9q, KP098017140 * T9d);
+		    T9L = FMA(KP773010453, T9G, KP634393284 * T9F);
+		    T9v = FMA(KP098017140, T9q, KP995184726 * T9d);
+		    T9H = FNMS(KP634393284, T9G, KP773010453 * T9F);
+	       }
+	       {
+		    E T7F, T9s, TiH, TiK;
+		    T7F = T6L + T7E;
+		    T9s = T8y + T9r;
+		    ri[WS(ios, 47)] = T7F - T9s;
+		    ri[WS(ios, 15)] = T7F + T9s;
+		    TiH = T9u + T9v;
+		    TiK = TiI + TiJ;
+		    ii[WS(ios, 15)] = TiH + TiK;
+		    ii[WS(ios, 47)] = TiK - TiH;
+	       }
+	       {
+		    E T9t, T9w, TiL, TiM;
+		    T9t = T6L - T7E;
+		    T9w = T9u - T9v;
+		    ri[WS(ios, 63)] = T9t - T9w;
+		    ri[WS(ios, 31)] = T9t + T9w;
+		    TiL = T9r - T8y;
+		    TiM = TiJ - TiI;
+		    ii[WS(ios, 31)] = TiL + TiM;
+		    ii[WS(ios, 63)] = TiM - TiL;
+	       }
+	       {
+		    E T9B, T9I, Tiz, TiE;
+		    T9B = T9x + T9A;
+		    T9I = T9E + T9H;
+		    ri[WS(ios, 39)] = T9B - T9I;
+		    ri[WS(ios, 7)] = T9B + T9I;
+		    Tiz = T9K + T9L;
+		    TiE = TiA + TiD;
+		    ii[WS(ios, 7)] = Tiz + TiE;
+		    ii[WS(ios, 39)] = TiE - Tiz;
+	       }
+	       {
+		    E T9J, T9M, TiF, TiG;
+		    T9J = T9x - T9A;
+		    T9M = T9K - T9L;
+		    ri[WS(ios, 55)] = T9J - T9M;
+		    ri[WS(ios, 23)] = T9J + T9M;
+		    TiF = T9H - T9E;
+		    TiG = TiD - TiA;
+		    ii[WS(ios, 23)] = TiF + TiG;
+		    ii[WS(ios, 55)] = TiG - TiF;
+	       }
+	  }
+	  {
+	       E TaL, TbJ, Ti9, Tif, Tb0, Tie, TbM, Ti6, Tbk, TbW, TbG, TbQ, TbD, TbX, TbH;
+	       E TbT;
+	       {
+		    E TaD, TaK, Ti7, Ti8;
+		    TaD = Taz - TaC;
+		    TaK = TaG - TaJ;
+		    TaL = TaD - TaK;
+		    TbJ = TaD + TaK;
+		    Ti7 = Tc1 - Tc0;
+		    Ti8 = ThT - ThQ;
+		    Ti9 = Ti7 + Ti8;
+		    Tif = Ti8 - Ti7;
+	       }
+	       {
+		    E TaS, TbK, TaZ, TbL;
+		    {
+			 E TaO, TaR, TaV, TaY;
+			 TaO = TaM - TaN;
+			 TaR = TaP - TaQ;
+			 TaS = FNMS(KP831469612, TaR, KP555570233 * TaO);
+			 TbK = FMA(KP555570233, TaR, KP831469612 * TaO);
+			 TaV = TaT - TaU;
+			 TaY = TaW - TaX;
+			 TaZ = FMA(KP831469612, TaV, KP555570233 * TaY);
+			 TbL = FNMS(KP831469612, TaY, KP555570233 * TaV);
+		    }
+		    Tb0 = TaS - TaZ;
+		    Tie = TbL - TbK;
+		    TbM = TbK + TbL;
+		    Ti6 = TaS + TaZ;
+	       }
+	       {
+		    E Tbc, TbO, Tbj, TbP;
+		    {
+			 E Tb4, Tbb, Tbf, Tbi;
+			 Tb4 = Tb2 - Tb3;
+			 Tbb = Tb7 - Tba;
+			 Tbc = Tb4 - Tbb;
+			 TbO = Tb4 + Tbb;
+			 Tbf = Tbd - Tbe;
+			 Tbi = Tbg - Tbh;
+			 Tbj = Tbf - Tbi;
+			 TbP = Tbf + Tbi;
+		    }
+		    Tbk = FMA(KP956940335, Tbc, KP290284677 * Tbj);
+		    TbW = FNMS(KP471396736, TbP, KP881921264 * TbO);
+		    TbG = FNMS(KP956940335, Tbj, KP290284677 * Tbc);
+		    TbQ = FMA(KP471396736, TbO, KP881921264 * TbP);
+	       }
+	       {
+		    E Tbv, TbR, TbC, TbS;
+		    {
+			 E Tbn, Tbu, Tby, TbB;
+			 Tbn = Tbl - Tbm;
+			 Tbu = Tbq - Tbt;
+			 Tbv = Tbn - Tbu;
+			 TbR = Tbn + Tbu;
+			 Tby = Tbw - Tbx;
+			 TbB = Tbz - TbA;
+			 TbC = Tby - TbB;
+			 TbS = Tby + TbB;
+		    }
+		    TbD = FNMS(KP956940335, TbC, KP290284677 * Tbv);
+		    TbX = FMA(KP881921264, TbS, KP471396736 * TbR);
+		    TbH = FMA(KP290284677, TbC, KP956940335 * Tbv);
+		    TbT = FNMS(KP471396736, TbS, KP881921264 * TbR);
+	       }
+	       {
+		    E Tb1, TbE, Tid, Tig;
+		    Tb1 = TaL + Tb0;
+		    TbE = Tbk + TbD;
+		    ri[WS(ios, 45)] = Tb1 - TbE;
+		    ri[WS(ios, 13)] = Tb1 + TbE;
+		    Tid = TbG + TbH;
+		    Tig = Tie + Tif;
+		    ii[WS(ios, 13)] = Tid + Tig;
+		    ii[WS(ios, 45)] = Tig - Tid;
+	       }
+	       {
+		    E TbF, TbI, Tih, Tii;
+		    TbF = TaL - Tb0;
+		    TbI = TbG - TbH;
+		    ri[WS(ios, 61)] = TbF - TbI;
+		    ri[WS(ios, 29)] = TbF + TbI;
+		    Tih = TbD - Tbk;
+		    Tii = Tif - Tie;
+		    ii[WS(ios, 29)] = Tih + Tii;
+		    ii[WS(ios, 61)] = Tii - Tih;
+	       }
+	       {
+		    E TbN, TbU, Ti5, Tia;
+		    TbN = TbJ + TbM;
+		    TbU = TbQ + TbT;
+		    ri[WS(ios, 37)] = TbN - TbU;
+		    ri[WS(ios, 5)] = TbN + TbU;
+		    Ti5 = TbW + TbX;
+		    Tia = Ti6 + Ti9;
+		    ii[WS(ios, 5)] = Ti5 + Tia;
+		    ii[WS(ios, 37)] = Tia - Ti5;
+	       }
+	       {
+		    E TbV, TbY, Tib, Tic;
+		    TbV = TbJ - TbM;
+		    TbY = TbW - TbX;
+		    ri[WS(ios, 53)] = TbV - TbY;
+		    ri[WS(ios, 21)] = TbV + TbY;
+		    Tib = TbT - TbQ;
+		    Tic = Ti9 - Ti6;
+		    ii[WS(ios, 21)] = Tib + Tic;
+		    ii[WS(ios, 53)] = Tic - Tib;
+	       }
+	  }
+	  {
+	       E Tc3, Tcv, ThV, Ti1, Tca, Ti0, Tcy, ThO, Tci, TcI, Tcs, TcC, Tcp, TcJ, Tct;
+	       E TcF;
+	       {
+		    E TbZ, Tc2, ThP, ThU;
+		    TbZ = Taz + TaC;
+		    Tc2 = Tc0 + Tc1;
+		    Tc3 = TbZ - Tc2;
+		    Tcv = TbZ + Tc2;
+		    ThP = TaG + TaJ;
+		    ThU = ThQ + ThT;
+		    ThV = ThP + ThU;
+		    Ti1 = ThU - ThP;
+	       }
+	       {
+		    E Tc6, Tcw, Tc9, Tcx;
+		    {
+			 E Tc4, Tc5, Tc7, Tc8;
+			 Tc4 = TaM + TaN;
+			 Tc5 = TaP + TaQ;
+			 Tc6 = FNMS(KP195090322, Tc5, KP980785280 * Tc4);
+			 Tcw = FMA(KP980785280, Tc5, KP195090322 * Tc4);
+			 Tc7 = TaT + TaU;
+			 Tc8 = TaW + TaX;
+			 Tc9 = FMA(KP195090322, Tc7, KP980785280 * Tc8);
+			 Tcx = FNMS(KP195090322, Tc8, KP980785280 * Tc7);
+		    }
+		    Tca = Tc6 - Tc9;
+		    Ti0 = Tcx - Tcw;
+		    Tcy = Tcw + Tcx;
+		    ThO = Tc6 + Tc9;
+	       }
+	       {
+		    E Tce, TcA, Tch, TcB;
+		    {
+			 E Tcc, Tcd, Tcf, Tcg;
+			 Tcc = Tbd + Tbe;
+			 Tcd = Tba + Tb7;
+			 Tce = Tcc - Tcd;
+			 TcA = Tcc + Tcd;
+			 Tcf = Tb2 + Tb3;
+			 Tcg = Tbg + Tbh;
+			 Tch = Tcf - Tcg;
+			 TcB = Tcf + Tcg;
+		    }
+		    Tci = FMA(KP634393284, Tce, KP773010453 * Tch);
+		    TcI = FNMS(KP098017140, TcA, KP995184726 * TcB);
+		    Tcs = FNMS(KP773010453, Tce, KP634393284 * Tch);
+		    TcC = FMA(KP995184726, TcA, KP098017140 * TcB);
+	       }
+	       {
+		    E Tcl, TcD, Tco, TcE;
+		    {
+			 E Tcj, Tck, Tcm, Tcn;
+			 Tcj = Tbl + Tbm;
+			 Tck = TbA + Tbz;
+			 Tcl = Tcj - Tck;
+			 TcD = Tcj + Tck;
+			 Tcm = Tbw + Tbx;
+			 Tcn = Tbq + Tbt;
+			 Tco = Tcm - Tcn;
+			 TcE = Tcm + Tcn;
+		    }
+		    Tcp = FNMS(KP773010453, Tco, KP634393284 * Tcl);
+		    TcJ = FMA(KP098017140, TcD, KP995184726 * TcE);
+		    Tct = FMA(KP773010453, Tcl, KP634393284 * Tco);
+		    TcF = FNMS(KP098017140, TcE, KP995184726 * TcD);
+	       }
+	       {
+		    E Tcb, Tcq, ThZ, Ti2;
+		    Tcb = Tc3 + Tca;
+		    Tcq = Tci + Tcp;
+		    ri[WS(ios, 41)] = Tcb - Tcq;
+		    ri[WS(ios, 9)] = Tcb + Tcq;
+		    ThZ = Tcs + Tct;
+		    Ti2 = Ti0 + Ti1;
+		    ii[WS(ios, 9)] = ThZ + Ti2;
+		    ii[WS(ios, 41)] = Ti2 - ThZ;
+	       }
+	       {
+		    E Tcr, Tcu, Ti3, Ti4;
+		    Tcr = Tc3 - Tca;
+		    Tcu = Tcs - Tct;
+		    ri[WS(ios, 57)] = Tcr - Tcu;
+		    ri[WS(ios, 25)] = Tcr + Tcu;
+		    Ti3 = Tcp - Tci;
+		    Ti4 = Ti1 - Ti0;
+		    ii[WS(ios, 25)] = Ti3 + Ti4;
+		    ii[WS(ios, 57)] = Ti4 - Ti3;
+	       }
+	       {
+		    E Tcz, TcG, ThN, ThW;
+		    Tcz = Tcv + Tcy;
+		    TcG = TcC + TcF;
+		    ri[WS(ios, 33)] = Tcz - TcG;
+		    ri[WS(ios, 1)] = Tcz + TcG;
+		    ThN = TcI + TcJ;
+		    ThW = ThO + ThV;
+		    ii[WS(ios, 1)] = ThN + ThW;
+		    ii[WS(ios, 33)] = ThW - ThN;
+	       }
+	       {
+		    E TcH, TcK, ThX, ThY;
+		    TcH = Tcv - Tcy;
+		    TcK = TcI - TcJ;
+		    ri[WS(ios, 49)] = TcH - TcK;
+		    ri[WS(ios, 17)] = TcH + TcK;
+		    ThX = TcF - TcC;
+		    ThY = ThV - ThO;
+		    ii[WS(ios, 17)] = ThX + ThY;
+		    ii[WS(ios, 49)] = ThY - ThX;
+	       }
+	  }
+	  {
+	       E T9R, Taj, Tip, Tiv, T9Y, Tiu, Tam, Tik, Ta6, Taw, Tag, Taq, Tad, Tax, Tah;
+	       E Tat;
+	       {
+		    E T9N, T9Q, Til, Tio;
+		    T9N = T6b + T6m;
+		    T9Q = T9O + T9P;
+		    T9R = T9N - T9Q;
+		    Taj = T9N + T9Q;
+		    Til = T6y + T6J;
+		    Tio = Tim + Tin;
+		    Tip = Til + Tio;
+		    Tiv = Tio - Til;
+	       }
+	       {
+		    E T9U, Tak, T9X, Tal;
+		    {
+			 E T9S, T9T, T9V, T9W;
+			 T9S = T6Q + T71;
+			 T9T = T77 + T7a;
+			 T9U = FNMS(KP555570233, T9T, KP831469612 * T9S);
+			 Tak = FMA(KP555570233, T9S, KP831469612 * T9T);
+			 T9V = T7h + T7s;
+			 T9W = T7y + T7B;
+			 T9X = FMA(KP831469612, T9V, KP555570233 * T9W);
+			 Tal = FNMS(KP555570233, T9V, KP831469612 * T9W);
+		    }
+		    T9Y = T9U - T9X;
+		    Tiu = Tal - Tak;
+		    Tam = Tak + Tal;
+		    Tik = T9U + T9X;
+	       }
+	       {
+		    E Ta2, Tao, Ta5, Tap;
+		    {
+			 E Ta0, Ta1, Ta3, Ta4;
+			 Ta0 = T8p + T8s;
+			 Ta1 = T8i + T87;
+			 Ta2 = Ta0 - Ta1;
+			 Tao = Ta0 + Ta1;
+			 Ta3 = T7K + T7V;
+			 Ta4 = T8u + T8v;
+			 Ta5 = Ta3 - Ta4;
+			 Tap = Ta3 + Ta4;
+		    }
+		    Ta6 = FMA(KP471396736, Ta2, KP881921264 * Ta5);
+		    Taw = FNMS(KP290284677, Tao, KP956940335 * Tap);
+		    Tag = FNMS(KP881921264, Ta2, KP471396736 * Ta5);
+		    Taq = FMA(KP956940335, Tao, KP290284677 * Tap);
+	       }
+	       {
+		    E Ta9, Tar, Tac, Tas;
+		    {
+			 E Ta7, Ta8, Taa, Tab;
+			 Ta7 = T8D + T8O;
+			 Ta8 = T9o + T9n;
+			 Ta9 = Ta7 - Ta8;
+			 Tar = Ta7 + Ta8;
+			 Taa = T9i + T9l;
+			 Tab = T90 + T9b;
+			 Tac = Taa - Tab;
+			 Tas = Taa + Tab;
+		    }
+		    Tad = FNMS(KP881921264, Tac, KP471396736 * Ta9);
+		    Tax = FMA(KP290284677, Tar, KP956940335 * Tas);
+		    Tah = FMA(KP881921264, Ta9, KP471396736 * Tac);
+		    Tat = FNMS(KP290284677, Tas, KP956940335 * Tar);
+	       }
+	       {
+		    E T9Z, Tae, Tit, Tiw;
+		    T9Z = T9R + T9Y;
+		    Tae = Ta6 + Tad;
+		    ri[WS(ios, 43)] = T9Z - Tae;
+		    ri[WS(ios, 11)] = T9Z + Tae;
+		    Tit = Tag + Tah;
+		    Tiw = Tiu + Tiv;
+		    ii[WS(ios, 11)] = Tit + Tiw;
+		    ii[WS(ios, 43)] = Tiw - Tit;
+	       }
+	       {
+		    E Taf, Tai, Tix, Tiy;
+		    Taf = T9R - T9Y;
+		    Tai = Tag - Tah;
+		    ri[WS(ios, 59)] = Taf - Tai;
+		    ri[WS(ios, 27)] = Taf + Tai;
+		    Tix = Tad - Ta6;
+		    Tiy = Tiv - Tiu;
+		    ii[WS(ios, 27)] = Tix + Tiy;
+		    ii[WS(ios, 59)] = Tiy - Tix;
+	       }
+	       {
+		    E Tan, Tau, Tij, Tiq;
+		    Tan = Taj + Tam;
+		    Tau = Taq + Tat;
+		    ri[WS(ios, 35)] = Tan - Tau;
+		    ri[WS(ios, 3)] = Tan + Tau;
+		    Tij = Taw + Tax;
+		    Tiq = Tik + Tip;
+		    ii[WS(ios, 3)] = Tij + Tiq;
+		    ii[WS(ios, 35)] = Tiq - Tij;
+	       }
+	       {
+		    E Tav, Tay, Tir, Tis;
+		    Tav = Taj - Tam;
+		    Tay = Taw - Tax;
+		    ri[WS(ios, 51)] = Tav - Tay;
+		    ri[WS(ios, 19)] = Tav + Tay;
+		    Tir = Tat - Taq;
+		    Tis = Tip - Tik;
+		    ii[WS(ios, 19)] = Tir + Tis;
+		    ii[WS(ios, 51)] = Tis - Tir;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 64},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 64, "t1_64", twinstr, {808, 270, 230, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_64) (planner *p) {
+     X(kdft_dit_register) (p, t1_64, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_7.c b/src/fftw3/dft/codelets/standard/t1_7.c
new file mode 100644
index 0000000..d52c5eb
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_7.c
@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:56 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 7 -name t1_7 -include t.h */
+
+/*
+ * This function contains 72 FP additions, 60 FP multiplications,
+ * (or, 36 additions, 24 multiplications, 36 fused multiply/add),
+ * 29 stack variables, and 28 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_7.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_7.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_7.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_7(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP222520933, +0.222520933956314404288902564496794759466355569);
+     DK(KP900968867, +0.900968867902419126236102319507445051165919162);
+     DK(KP623489801, +0.623489801858733530525004884004239810632274731);
+     DK(KP433883739, +0.433883739117558120475768332848358754609990728);
+     DK(KP781831482, +0.781831482468029808708444526674057750232334519);
+     DK(KP974927912, +0.974927912181823607018131682993931217232785801);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 12) {
+	  E T1, TR, Tc, TS, TC, TO, Tn, TT, TI, TP, Ty, TU, TF, TQ;
+	  T1 = ri[0];
+	  TR = ii[0];
+	  {
+	       E T6, TA, Tb, TB;
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = ri[WS(ios, 1)];
+		    T5 = ii[WS(ios, 1)];
+		    T2 = W[0];
+		    T4 = W[1];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    TA = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T8, Ta, T7, T9;
+		    T8 = ri[WS(ios, 6)];
+		    Ta = ii[WS(ios, 6)];
+		    T7 = W[10];
+		    T9 = W[11];
+		    Tb = FMA(T7, T8, T9 * Ta);
+		    TB = FNMS(T9, T8, T7 * Ta);
+	       }
+	       Tc = T6 + Tb;
+	       TS = Tb - T6;
+	       TC = TA - TB;
+	       TO = TA + TB;
+	  }
+	  {
+	       E Th, TG, Tm, TH;
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = ri[WS(ios, 2)];
+		    Tg = ii[WS(ios, 2)];
+		    Td = W[2];
+		    Tf = W[3];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    TG = FNMS(Tf, Te, Td * Tg);
+	       }
+	       {
+		    E Tj, Tl, Ti, Tk;
+		    Tj = ri[WS(ios, 5)];
+		    Tl = ii[WS(ios, 5)];
+		    Ti = W[8];
+		    Tk = W[9];
+		    Tm = FMA(Ti, Tj, Tk * Tl);
+		    TH = FNMS(Tk, Tj, Ti * Tl);
+	       }
+	       Tn = Th + Tm;
+	       TT = Tm - Th;
+	       TI = TG - TH;
+	       TP = TG + TH;
+	  }
+	  {
+	       E Ts, TD, Tx, TE;
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = ri[WS(ios, 3)];
+		    Tr = ii[WS(ios, 3)];
+		    To = W[4];
+		    Tq = W[5];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    TD = FNMS(Tq, Tp, To * Tr);
+	       }
+	       {
+		    E Tu, Tw, Tt, Tv;
+		    Tu = ri[WS(ios, 4)];
+		    Tw = ii[WS(ios, 4)];
+		    Tt = W[6];
+		    Tv = W[7];
+		    Tx = FMA(Tt, Tu, Tv * Tw);
+		    TE = FNMS(Tv, Tu, Tt * Tw);
+	       }
+	       Ty = Ts + Tx;
+	       TU = Tx - Ts;
+	       TF = TD - TE;
+	       TQ = TD + TE;
+	  }
+	  ri[0] = T1 + Tc + Tn + Ty;
+	  ii[0] = TO + TP + TQ + TR;
+	  {
+	       E TJ, Tz, TX, TY;
+	       TJ = FNMS(KP781831482, TF, KP974927912 * TC) - (KP433883739 * TI);
+	       Tz = FMA(KP623489801, Ty, T1) + FNMA(KP900968867, Tn, KP222520933 * Tc);
+	       ri[WS(ios, 5)] = Tz - TJ;
+	       ri[WS(ios, 2)] = Tz + TJ;
+	       TX = FNMS(KP781831482, TU, KP974927912 * TS) - (KP433883739 * TT);
+	       TY = FMA(KP623489801, TQ, TR) + FNMA(KP900968867, TP, KP222520933 * TO);
+	       ii[WS(ios, 2)] = TX + TY;
+	       ii[WS(ios, 5)] = TY - TX;
+	  }
+	  {
+	       E TL, TK, TV, TW;
+	       TL = FMA(KP781831482, TC, KP974927912 * TI) + (KP433883739 * TF);
+	       TK = FMA(KP623489801, Tc, T1) + FNMA(KP900968867, Ty, KP222520933 * Tn);
+	       ri[WS(ios, 6)] = TK - TL;
+	       ri[WS(ios, 1)] = TK + TL;
+	       TV = FMA(KP781831482, TS, KP974927912 * TT) + (KP433883739 * TU);
+	       TW = FMA(KP623489801, TO, TR) + FNMA(KP900968867, TQ, KP222520933 * TP);
+	       ii[WS(ios, 1)] = TV + TW;
+	       ii[WS(ios, 6)] = TW - TV;
+	  }
+	  {
+	       E TN, TM, TZ, T10;
+	       TN = FMA(KP433883739, TC, KP974927912 * TF) - (KP781831482 * TI);
+	       TM = FMA(KP623489801, Tn, T1) + FNMA(KP222520933, Ty, KP900968867 * Tc);
+	       ri[WS(ios, 4)] = TM - TN;
+	       ri[WS(ios, 3)] = TM + TN;
+	       TZ = FMA(KP433883739, TS, KP974927912 * TU) - (KP781831482 * TT);
+	       T10 = FMA(KP623489801, TP, TR) + FNMA(KP222520933, TQ, KP900968867 * TO);
+	       ii[WS(ios, 3)] = TZ + T10;
+	       ii[WS(ios, 4)] = T10 - TZ;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 7},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 7, "t1_7", twinstr, {36, 24, 36, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_7) (planner *p) {
+     X(kdft_dit_register) (p, t1_7, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_8.c b/src/fftw3/dft/codelets/standard/t1_8.c
new file mode 100644
index 0000000..b9cb995
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_8.c
@@ -0,0 +1,203 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:29:59 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 8 -name t1_8 -include t.h */
+
+/*
+ * This function contains 66 FP additions, 32 FP multiplications,
+ * (or, 52 additions, 18 multiplications, 14 fused multiply/add),
+ * 28 stack variables, and 32 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_8.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_8.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_8.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_8(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 14) {
+	  E T7, T1e, TH, T19, TF, T13, TR, TU, Ti, T1f, TK, T16, Tu, T12, TM;
+	  E TP;
+	  {
+	       E T1, T18, T6, T17;
+	       T1 = ri[0];
+	       T18 = ii[0];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = ri[WS(ios, 4)];
+		    T5 = ii[WS(ios, 4)];
+		    T2 = W[6];
+		    T4 = W[7];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T17 = FNMS(T4, T3, T2 * T5);
+	       }
+	       T7 = T1 + T6;
+	       T1e = T18 - T17;
+	       TH = T1 - T6;
+	       T19 = T17 + T18;
+	  }
+	  {
+	       E Tz, TS, TE, TT;
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = ri[WS(ios, 7)];
+		    Ty = ii[WS(ios, 7)];
+		    Tv = W[12];
+		    Tx = W[13];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    TS = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = ri[WS(ios, 3)];
+		    TD = ii[WS(ios, 3)];
+		    TA = W[4];
+		    TC = W[5];
+		    TE = FMA(TA, TB, TC * TD);
+		    TT = FNMS(TC, TB, TA * TD);
+	       }
+	       TF = Tz + TE;
+	       T13 = TS + TT;
+	       TR = Tz - TE;
+	       TU = TS - TT;
+	  }
+	  {
+	       E Tc, TI, Th, TJ;
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = ri[WS(ios, 2)];
+		    Tb = ii[WS(ios, 2)];
+		    T8 = W[2];
+		    Ta = W[3];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    TI = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = ri[WS(ios, 6)];
+		    Tg = ii[WS(ios, 6)];
+		    Td = W[10];
+		    Tf = W[11];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    TJ = FNMS(Tf, Te, Td * Tg);
+	       }
+	       Ti = Tc + Th;
+	       T1f = Tc - Th;
+	       TK = TI - TJ;
+	       T16 = TI + TJ;
+	  }
+	  {
+	       E To, TN, Tt, TO;
+	       {
+		    E Tl, Tn, Tk, Tm;
+		    Tl = ri[WS(ios, 1)];
+		    Tn = ii[WS(ios, 1)];
+		    Tk = W[0];
+		    Tm = W[1];
+		    To = FMA(Tk, Tl, Tm * Tn);
+		    TN = FNMS(Tm, Tl, Tk * Tn);
+	       }
+	       {
+		    E Tq, Ts, Tp, Tr;
+		    Tq = ri[WS(ios, 5)];
+		    Ts = ii[WS(ios, 5)];
+		    Tp = W[8];
+		    Tr = W[9];
+		    Tt = FMA(Tp, Tq, Tr * Ts);
+		    TO = FNMS(Tr, Tq, Tp * Ts);
+	       }
+	       Tu = To + Tt;
+	       T12 = TN + TO;
+	       TM = To - Tt;
+	       TP = TN - TO;
+	  }
+	  {
+	       E Tj, TG, T1b, T1c;
+	       Tj = T7 + Ti;
+	       TG = Tu + TF;
+	       ri[WS(ios, 4)] = Tj - TG;
+	       ri[0] = Tj + TG;
+	       {
+		    E T15, T1a, T11, T14;
+		    T15 = T12 + T13;
+		    T1a = T16 + T19;
+		    ii[0] = T15 + T1a;
+		    ii[WS(ios, 4)] = T1a - T15;
+		    T11 = T7 - Ti;
+		    T14 = T12 - T13;
+		    ri[WS(ios, 6)] = T11 - T14;
+		    ri[WS(ios, 2)] = T11 + T14;
+	       }
+	       T1b = TF - Tu;
+	       T1c = T19 - T16;
+	       ii[WS(ios, 2)] = T1b + T1c;
+	       ii[WS(ios, 6)] = T1c - T1b;
+	       {
+		    E TX, T1g, T10, T1d, TY, TZ;
+		    TX = TH - TK;
+		    T1g = T1e - T1f;
+		    TY = TP - TM;
+		    TZ = TR + TU;
+		    T10 = KP707106781 * (TY - TZ);
+		    T1d = KP707106781 * (TY + TZ);
+		    ri[WS(ios, 7)] = TX - T10;
+		    ii[WS(ios, 5)] = T1g - T1d;
+		    ri[WS(ios, 3)] = TX + T10;
+		    ii[WS(ios, 1)] = T1d + T1g;
+	       }
+	       {
+		    E TL, T1i, TW, T1h, TQ, TV;
+		    TL = TH + TK;
+		    T1i = T1f + T1e;
+		    TQ = TM + TP;
+		    TV = TR - TU;
+		    TW = KP707106781 * (TQ + TV);
+		    T1h = KP707106781 * (TV - TQ);
+		    ri[WS(ios, 5)] = TL - TW;
+		    ii[WS(ios, 7)] = T1i - T1h;
+		    ri[WS(ios, 1)] = TL + TW;
+		    ii[WS(ios, 3)] = T1h + T1i;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 8},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 8, "t1_8", twinstr, {52, 18, 14, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_8) (planner *p) {
+     X(kdft_dit_register) (p, t1_8, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t1_9.c b/src/fftw3/dft/codelets/standard/t1_9.c
new file mode 100644
index 0000000..924e456
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t1_9.c
@@ -0,0 +1,256 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:00 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -n 9 -name t1_9 -include t.h */
+
+/*
+ * This function contains 96 FP additions, 72 FP multiplications,
+ * (or, 60 additions, 36 multiplications, 36 fused multiply/add),
+ * 41 stack variables, and 36 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t1_9.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_9.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t1_9.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t1_9(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP939692620, +0.939692620785908384054109277324731469936208134);
+     DK(KP342020143, +0.342020143325668733044099614682259580763083368);
+     DK(KP984807753, +0.984807753012208059366743024589523013670643252);
+     DK(KP173648177, +0.173648177666930348851716626769314796000375677);
+     DK(KP642787609, +0.642787609686539326322643409907263432907559884);
+     DK(KP766044443, +0.766044443118978035202392650555416673935832457);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 16) {
+	  E T1, T1B, TQ, T1G, Tc, TN, T1A, T1H, TL, T1x, T17, T1o, T1c, T1n, Tu;
+	  E T1w, TW, T1k, T11, T1l;
+	  {
+	       E T6, TO, Tb, TP;
+	       T1 = ri[0];
+	       T1B = ii[0];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = ri[WS(ios, 3)];
+		    T5 = ii[WS(ios, 3)];
+		    T2 = W[4];
+		    T4 = W[5];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    TO = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T8, Ta, T7, T9;
+		    T8 = ri[WS(ios, 6)];
+		    Ta = ii[WS(ios, 6)];
+		    T7 = W[10];
+		    T9 = W[11];
+		    Tb = FMA(T7, T8, T9 * Ta);
+		    TP = FNMS(T9, T8, T7 * Ta);
+	       }
+	       TQ = KP866025403 * (TO - TP);
+	       T1G = KP866025403 * (Tb - T6);
+	       Tc = T6 + Tb;
+	       TN = FNMS(KP500000000, Tc, T1);
+	       T1A = TO + TP;
+	       T1H = FNMS(KP500000000, T1A, T1B);
+	  }
+	  {
+	       E Tz, T19, TE, T14, TJ, T15, TK, T1a;
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = ri[WS(ios, 2)];
+		    Ty = ii[WS(ios, 2)];
+		    Tv = W[2];
+		    Tx = W[3];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T19 = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = ri[WS(ios, 5)];
+		    TD = ii[WS(ios, 5)];
+		    TA = W[8];
+		    TC = W[9];
+		    TE = FMA(TA, TB, TC * TD);
+		    T14 = FNMS(TC, TB, TA * TD);
+	       }
+	       {
+		    E TG, TI, TF, TH;
+		    TG = ri[WS(ios, 8)];
+		    TI = ii[WS(ios, 8)];
+		    TF = W[14];
+		    TH = W[15];
+		    TJ = FMA(TF, TG, TH * TI);
+		    T15 = FNMS(TH, TG, TF * TI);
+	       }
+	       TK = TE + TJ;
+	       T1a = T14 + T15;
+	       TL = Tz + TK;
+	       T1x = T19 + T1a;
+	       {
+		    E T13, T16, T18, T1b;
+		    T13 = FNMS(KP500000000, TK, Tz);
+		    T16 = KP866025403 * (T14 - T15);
+		    T17 = T13 + T16;
+		    T1o = T13 - T16;
+		    T18 = KP866025403 * (TJ - TE);
+		    T1b = FNMS(KP500000000, T1a, T19);
+		    T1c = T18 + T1b;
+		    T1n = T1b - T18;
+	       }
+	  }
+	  {
+	       E Ti, TY, Tn, TT, Ts, TU, Tt, TZ;
+	       {
+		    E Tf, Th, Te, Tg;
+		    Tf = ri[WS(ios, 1)];
+		    Th = ii[WS(ios, 1)];
+		    Te = W[0];
+		    Tg = W[1];
+		    Ti = FMA(Te, Tf, Tg * Th);
+		    TY = FNMS(Tg, Tf, Te * Th);
+	       }
+	       {
+		    E Tk, Tm, Tj, Tl;
+		    Tk = ri[WS(ios, 4)];
+		    Tm = ii[WS(ios, 4)];
+		    Tj = W[6];
+		    Tl = W[7];
+		    Tn = FMA(Tj, Tk, Tl * Tm);
+		    TT = FNMS(Tl, Tk, Tj * Tm);
+	       }
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = ri[WS(ios, 7)];
+		    Tr = ii[WS(ios, 7)];
+		    To = W[12];
+		    Tq = W[13];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    TU = FNMS(Tq, Tp, To * Tr);
+	       }
+	       Tt = Tn + Ts;
+	       TZ = TT + TU;
+	       Tu = Ti + Tt;
+	       T1w = TY + TZ;
+	       {
+		    E TS, TV, TX, T10;
+		    TS = FNMS(KP500000000, Tt, Ti);
+		    TV = KP866025403 * (TT - TU);
+		    TW = TS + TV;
+		    T1k = TS - TV;
+		    TX = KP866025403 * (Ts - Tn);
+		    T10 = FNMS(KP500000000, TZ, TY);
+		    T11 = TX + T10;
+		    T1l = T10 - TX;
+	       }
+	  }
+	  {
+	       E T1y, Td, TM, T1v;
+	       T1y = KP866025403 * (T1w - T1x);
+	       Td = T1 + Tc;
+	       TM = Tu + TL;
+	       T1v = FNMS(KP500000000, TM, Td);
+	       ri[0] = Td + TM;
+	       ri[WS(ios, 3)] = T1v + T1y;
+	       ri[WS(ios, 6)] = T1v - T1y;
+	  }
+	  {
+	       E T1D, T1z, T1C, T1E;
+	       T1D = KP866025403 * (TL - Tu);
+	       T1z = T1w + T1x;
+	       T1C = T1A + T1B;
+	       T1E = FNMS(KP500000000, T1z, T1C);
+	       ii[0] = T1z + T1C;
+	       ii[WS(ios, 6)] = T1E - T1D;
+	       ii[WS(ios, 3)] = T1D + T1E;
+	  }
+	  {
+	       E TR, T1I, T1e, T1J, T1i, T1F, T1f, T1K;
+	       TR = TN + TQ;
+	       T1I = T1G + T1H;
+	       {
+		    E T12, T1d, T1g, T1h;
+		    T12 = FMA(KP766044443, TW, KP642787609 * T11);
+		    T1d = FMA(KP173648177, T17, KP984807753 * T1c);
+		    T1e = T12 + T1d;
+		    T1J = KP866025403 * (T1d - T12);
+		    T1g = FNMS(KP642787609, TW, KP766044443 * T11);
+		    T1h = FNMS(KP984807753, T17, KP173648177 * T1c);
+		    T1i = KP866025403 * (T1g - T1h);
+		    T1F = T1g + T1h;
+	       }
+	       ri[WS(ios, 1)] = TR + T1e;
+	       ii[WS(ios, 1)] = T1F + T1I;
+	       T1f = FNMS(KP500000000, T1e, TR);
+	       ri[WS(ios, 7)] = T1f - T1i;
+	       ri[WS(ios, 4)] = T1f + T1i;
+	       T1K = FNMS(KP500000000, T1F, T1I);
+	       ii[WS(ios, 4)] = T1J + T1K;
+	       ii[WS(ios, 7)] = T1K - T1J;
+	  }
+	  {
+	       E T1j, T1M, T1q, T1N, T1u, T1L, T1r, T1O;
+	       T1j = TN - TQ;
+	       T1M = T1H - T1G;
+	       {
+		    E T1m, T1p, T1s, T1t;
+		    T1m = FMA(KP173648177, T1k, KP984807753 * T1l);
+		    T1p = FNMS(KP939692620, T1o, KP342020143 * T1n);
+		    T1q = T1m + T1p;
+		    T1N = KP866025403 * (T1p - T1m);
+		    T1s = FNMS(KP984807753, T1k, KP173648177 * T1l);
+		    T1t = FMA(KP342020143, T1o, KP939692620 * T1n);
+		    T1u = KP866025403 * (T1s + T1t);
+		    T1L = T1s - T1t;
+	       }
+	       ri[WS(ios, 2)] = T1j + T1q;
+	       ii[WS(ios, 2)] = T1L + T1M;
+	       T1r = FNMS(KP500000000, T1q, T1j);
+	       ri[WS(ios, 8)] = T1r - T1u;
+	       ri[WS(ios, 5)] = T1r + T1u;
+	       T1O = FNMS(KP500000000, T1L, T1M);
+	       ii[WS(ios, 5)] = T1N + T1O;
+	       ii[WS(ios, 8)] = T1O - T1N;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 9},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 9, "t1_9", twinstr, {60, 36, 36, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t1_9) (planner *p) {
+     X(kdft_dit_register) (p, t1_9, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t2_16.c b/src/fftw3/dft/codelets/standard/t2_16.c
new file mode 100644
index 0000000..46d4bdb
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t2_16.c
@@ -0,0 +1,411 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:12 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -twiddle-log3 -n 16 -name t2_16 -include t.h */
+
+/*
+ * This function contains 196 FP additions, 108 FP multiplications,
+ * (or, 156 additions, 68 multiplications, 40 fused multiply/add),
+ * 104 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t2_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t2_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t2_16.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t2_16(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 8) {
+	  E T1, T3d, T18, T26, T29, T2R, Tq, T1r, T1E, T2k, T2g, T1O, Te, T3c, Tz;
+	  E T1P, T1S, T1T, T1U, TG, TL, T1V, T1Y, T1Z, T20, TT, TY, T1X, T1A, T2l;
+	  E T1J, T2h, T1h, T2b, T1m, T2a;
+	  T1 = ri[0];
+	  T3d = ii[0];
+	  {
+	       E T9, Td, Tl, Tp, Ty, Tu, TD, TF, TI, TK, TV, TQ, TS, TX, T1z;
+	       E T1v, T1C, T1D, T1G, T1I, T1q, T1p, T1l, T1j, T1c, T1g, T2, T5, Ti, Tg;
+	       E T4, Tw, Ts, Ta, Tv, T7, Tb, Tr, Tk, TW, TJ, TC, TU, To, TE;
+	       E TH, T14, T24, T17, T25, TN, TO, TP, TR;
+	       T9 = ri[WS(ios, 8)];
+	       Td = ii[WS(ios, 8)];
+	       Tl = ri[WS(ios, 4)];
+	       Tp = ii[WS(ios, 4)];
+	       Ty = ii[WS(ios, 12)];
+	       Tu = ri[WS(ios, 12)];
+	       TD = ri[WS(ios, 2)];
+	       TF = ii[WS(ios, 2)];
+	       TI = ri[WS(ios, 10)];
+	       TK = ii[WS(ios, 10)];
+	       TV = ri[WS(ios, 6)];
+	       TQ = ri[WS(ios, 14)];
+	       TS = ii[WS(ios, 14)];
+	       TX = ii[WS(ios, 6)];
+	       T1z = ii[WS(ios, 7)];
+	       T1v = ri[WS(ios, 7)];
+	       T1C = ri[WS(ios, 3)];
+	       T1D = ii[WS(ios, 3)];
+	       T1G = ri[WS(ios, 11)];
+	       T1I = ii[WS(ios, 11)];
+	       T1q = ii[WS(ios, 15)];
+	       T1p = ri[WS(ios, 15)];
+	       T1l = ii[WS(ios, 13)];
+	       T1j = ri[WS(ios, 13)];
+	       T1c = ri[WS(ios, 5)];
+	       T1g = ii[WS(ios, 5)];
+	       {
+		    E T12, T13, T15, T16, T3, T6, Tm, Tj, Tn, Th;
+		    T12 = ri[WS(ios, 1)];
+		    T13 = ii[WS(ios, 1)];
+		    T15 = ri[WS(ios, 9)];
+		    T16 = ii[WS(ios, 9)];
+		    T2 = W[4];
+		    T5 = W[5];
+		    T3 = W[0];
+		    T6 = W[1];
+		    Ti = W[3];
+		    Tg = W[2];
+		    T4 = T2 * T3;
+		    Tw = T5 * Tg;
+		    Ts = T5 * Ti;
+		    Ta = T2 * T6;
+		    Tv = T2 * Ti;
+		    T7 = T5 * T6;
+		    Tb = T5 * T3;
+		    Tr = T2 * Tg;
+		    Tm = Tg * T6;
+		    Tj = Ti * T6;
+		    Tn = Ti * T3;
+		    Th = Tg * T3;
+		    Tk = Th - Tj;
+		    TW = Tv - Tw;
+		    TJ = Ta + Tb;
+		    TC = Th + Tj;
+		    TU = Tr + Ts;
+		    To = Tm + Tn;
+		    TE = Tm - Tn;
+		    TH = T4 - T7;
+		    T14 = FMA(T3, T12, T6 * T13);
+		    T24 = FNMS(T6, T12, T3 * T13);
+		    T17 = FMA(T2, T15, T5 * T16);
+		    T25 = FNMS(T5, T15, T2 * T16);
+		    TN = W[6];
+		    TO = W[7];
+		    TP = FMA(TN, T3, TO * T6);
+		    TR = FNMS(TO, T3, TN * T6);
+	       }
+	       T18 = T14 + T17;
+	       T26 = T24 - T25;
+	       T29 = T14 - T17;
+	       T2R = T24 + T25;
+	       Tq = FMA(Tk, Tl, To * Tp);
+	       T1r = FMA(TN, T1p, TO * T1q);
+	       T1E = FMA(Tg, T1C, Ti * T1D);
+	       T2k = FNMS(TO, T1p, TN * T1q);
+	       T2g = FNMS(Ti, T1C, Tg * T1D);
+	       {
+		    E T8, Tc, Tt, Tx;
+		    T1O = FNMS(To, Tl, Tk * Tp);
+		    T8 = T4 + T7;
+		    Tc = Ta - Tb;
+		    Te = FNMS(Tc, Td, T8 * T9);
+		    T3c = FMA(Tc, T9, T8 * Td);
+		    Tt = Tr - Ts;
+		    Tx = Tv + Tw;
+		    Tz = FMA(Tt, Tu, Tx * Ty);
+		    T1P = FNMS(Tx, Tu, Tt * Ty);
+		    T1S = FMA(TE, TD, TC * TF);
+		    T1T = FNMS(TJ, TI, TH * TK);
+		    T1U = T1S - T1T;
+	       }
+	       TG = FNMS(TE, TF, TC * TD);
+	       TL = FMA(TH, TI, TJ * TK);
+	       T1V = TG - TL;
+	       T1Y = FMA(TR, TQ, TP * TS);
+	       T1Z = FMA(TW, TV, TU * TX);
+	       T20 = T1Y - T1Z;
+	       TT = FNMS(TR, TS, TP * TQ);
+	       TY = FNMS(TW, TX, TU * TV);
+	       T1X = TT - TY;
+	       {
+		    E T1u, T1F, T1y, T1H;
+		    {
+			 E T1s, T1t, T1w, T1x;
+			 T1s = T2 * TC;
+			 T1t = T5 * TE;
+			 T1u = T1s - T1t;
+			 T1F = T1s + T1t;
+			 T1w = T2 * TE;
+			 T1x = T5 * TC;
+			 T1y = T1w + T1x;
+			 T1H = T1w - T1x;
+		    }
+		    T1A = FMA(T1u, T1v, T1y * T1z);
+		    T2l = FNMS(T1y, T1v, T1u * T1z);
+		    T1J = FNMS(T1H, T1I, T1F * T1G);
+		    T2h = FMA(T1H, T1G, T1F * T1I);
+	       }
+	       {
+		    E T1b, T1i, T1f, T1k;
+		    {
+			 E T19, T1a, T1d, T1e;
+			 T19 = T2 * Tk;
+			 T1a = T5 * To;
+			 T1b = T19 + T1a;
+			 T1i = T19 - T1a;
+			 T1d = T2 * To;
+			 T1e = T5 * Tk;
+			 T1f = T1d - T1e;
+			 T1k = T1d + T1e;
+		    }
+		    T1h = FNMS(T1f, T1g, T1b * T1c);
+		    T2b = FNMS(T1k, T1j, T1i * T1l);
+		    T1m = FMA(T1i, T1j, T1k * T1l);
+		    T2a = FMA(T1f, T1c, T1b * T1g);
+	       }
+	  }
+	  {
+	       E TB, T2L, T10, T3k, T3f, T3l, T2O, T3a, T1o, T36, T2U, T32, T1L, T37, T2Z;
+	       E T33;
+	       {
+		    E Tf, TA, T2M, T2N;
+		    Tf = T1 + Te;
+		    TA = Tq + Tz;
+		    TB = Tf + TA;
+		    T2L = Tf - TA;
+		    {
+			 E TM, TZ, T3b, T3e;
+			 TM = TG + TL;
+			 TZ = TT + TY;
+			 T10 = TM + TZ;
+			 T3k = TZ - TM;
+			 T3b = T1O + T1P;
+			 T3e = T3c + T3d;
+			 T3f = T3b + T3e;
+			 T3l = T3e - T3b;
+		    }
+		    T2M = T1S + T1T;
+		    T2N = T1Y + T1Z;
+		    T2O = T2M - T2N;
+		    T3a = T2M + T2N;
+		    {
+			 E T1n, T2Q, T2S, T2T;
+			 T1n = T1h + T1m;
+			 T2Q = T18 - T1n;
+			 T2S = T2a + T2b;
+			 T2T = T2R - T2S;
+			 T1o = T18 + T1n;
+			 T36 = T2R + T2S;
+			 T2U = T2Q + T2T;
+			 T32 = T2T - T2Q;
+		    }
+		    {
+			 E T1B, T1K, T2V, T2W, T2X, T2Y;
+			 T1B = T1r + T1A;
+			 T1K = T1E + T1J;
+			 T2V = T1B - T1K;
+			 T2W = T2k + T2l;
+			 T2X = T2g + T2h;
+			 T2Y = T2W - T2X;
+			 T1L = T1B + T1K;
+			 T37 = T2W + T2X;
+			 T2Z = T2V - T2Y;
+			 T33 = T2V + T2Y;
+		    }
+	       }
+	       {
+		    E T11, T1M, T39, T3g;
+		    T11 = TB + T10;
+		    T1M = T1o + T1L;
+		    ri[WS(ios, 8)] = T11 - T1M;
+		    ri[0] = T11 + T1M;
+		    T39 = T36 + T37;
+		    T3g = T3a + T3f;
+		    ii[0] = T39 + T3g;
+		    ii[WS(ios, 8)] = T3g - T39;
+	       }
+	       {
+		    E T2P, T30, T3j, T3m;
+		    T2P = T2L + T2O;
+		    T30 = KP707106781 * (T2U + T2Z);
+		    ri[WS(ios, 10)] = T2P - T30;
+		    ri[WS(ios, 2)] = T2P + T30;
+		    T3j = KP707106781 * (T32 + T33);
+		    T3m = T3k + T3l;
+		    ii[WS(ios, 2)] = T3j + T3m;
+		    ii[WS(ios, 10)] = T3m - T3j;
+	       }
+	       {
+		    E T31, T34, T3n, T3o;
+		    T31 = T2L - T2O;
+		    T34 = KP707106781 * (T32 - T33);
+		    ri[WS(ios, 14)] = T31 - T34;
+		    ri[WS(ios, 6)] = T31 + T34;
+		    T3n = KP707106781 * (T2Z - T2U);
+		    T3o = T3l - T3k;
+		    ii[WS(ios, 6)] = T3n + T3o;
+		    ii[WS(ios, 14)] = T3o - T3n;
+	       }
+	       {
+		    E T35, T38, T3h, T3i;
+		    T35 = TB - T10;
+		    T38 = T36 - T37;
+		    ri[WS(ios, 12)] = T35 - T38;
+		    ri[WS(ios, 4)] = T35 + T38;
+		    T3h = T1L - T1o;
+		    T3i = T3f - T3a;
+		    ii[WS(ios, 4)] = T3h + T3i;
+		    ii[WS(ios, 12)] = T3i - T3h;
+	       }
+	  }
+	  {
+	       E T1R, T2v, T22, T3q, T3t, T3z, T2y, T3y, T2e, T2I, T2s, T2C, T2p, T2J, T2t;
+	       E T2F;
+	       {
+		    E T1N, T1Q, T2w, T2x;
+		    T1N = T1 - Te;
+		    T1Q = T1O - T1P;
+		    T1R = T1N - T1Q;
+		    T2v = T1N + T1Q;
+		    {
+			 E T1W, T21, T3r, T3s;
+			 T1W = T1U - T1V;
+			 T21 = T1X + T20;
+			 T22 = KP707106781 * (T1W - T21);
+			 T3q = KP707106781 * (T1W + T21);
+			 T3r = T3d - T3c;
+			 T3s = Tq - Tz;
+			 T3t = T3r - T3s;
+			 T3z = T3s + T3r;
+		    }
+		    T2w = T1V + T1U;
+		    T2x = T1X - T20;
+		    T2y = KP707106781 * (T2w + T2x);
+		    T3y = KP707106781 * (T2x - T2w);
+		    {
+			 E T28, T2A, T2d, T2B, T27, T2c;
+			 T27 = T1h - T1m;
+			 T28 = T26 + T27;
+			 T2A = T26 - T27;
+			 T2c = T2a - T2b;
+			 T2d = T29 - T2c;
+			 T2B = T29 + T2c;
+			 T2e = FMA(KP923879532, T28, KP382683432 * T2d);
+			 T2I = FNMS(KP382683432, T2B, KP923879532 * T2A);
+			 T2s = FNMS(KP923879532, T2d, KP382683432 * T28);
+			 T2C = FMA(KP382683432, T2A, KP923879532 * T2B);
+		    }
+		    {
+			 E T2j, T2D, T2o, T2E;
+			 {
+			      E T2f, T2i, T2m, T2n;
+			      T2f = T1r - T1A;
+			      T2i = T2g - T2h;
+			      T2j = T2f - T2i;
+			      T2D = T2f + T2i;
+			      T2m = T2k - T2l;
+			      T2n = T1E - T1J;
+			      T2o = T2m + T2n;
+			      T2E = T2m - T2n;
+			 }
+			 T2p = FNMS(KP923879532, T2o, KP382683432 * T2j);
+			 T2J = FMA(KP923879532, T2E, KP382683432 * T2D);
+			 T2t = FMA(KP382683432, T2o, KP923879532 * T2j);
+			 T2F = FNMS(KP382683432, T2E, KP923879532 * T2D);
+		    }
+	       }
+	       {
+		    E T23, T2q, T3x, T3A;
+		    T23 = T1R + T22;
+		    T2q = T2e + T2p;
+		    ri[WS(ios, 11)] = T23 - T2q;
+		    ri[WS(ios, 3)] = T23 + T2q;
+		    T3x = T2s + T2t;
+		    T3A = T3y + T3z;
+		    ii[WS(ios, 3)] = T3x + T3A;
+		    ii[WS(ios, 11)] = T3A - T3x;
+	       }
+	       {
+		    E T2r, T2u, T3B, T3C;
+		    T2r = T1R - T22;
+		    T2u = T2s - T2t;
+		    ri[WS(ios, 15)] = T2r - T2u;
+		    ri[WS(ios, 7)] = T2r + T2u;
+		    T3B = T2p - T2e;
+		    T3C = T3z - T3y;
+		    ii[WS(ios, 7)] = T3B + T3C;
+		    ii[WS(ios, 15)] = T3C - T3B;
+	       }
+	       {
+		    E T2z, T2G, T3p, T3u;
+		    T2z = T2v + T2y;
+		    T2G = T2C + T2F;
+		    ri[WS(ios, 9)] = T2z - T2G;
+		    ri[WS(ios, 1)] = T2z + T2G;
+		    T3p = T2I + T2J;
+		    T3u = T3q + T3t;
+		    ii[WS(ios, 1)] = T3p + T3u;
+		    ii[WS(ios, 9)] = T3u - T3p;
+	       }
+	       {
+		    E T2H, T2K, T3v, T3w;
+		    T2H = T2v - T2y;
+		    T2K = T2I - T2J;
+		    ri[WS(ios, 13)] = T2H - T2K;
+		    ri[WS(ios, 5)] = T2H + T2K;
+		    T3v = T2F - T2C;
+		    T3w = T3t - T3q;
+		    ii[WS(ios, 5)] = T3v + T3w;
+		    ii[WS(ios, 13)] = T3w - T3v;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_COS, 0, 1},
+     {TW_SIN, 0, 1},
+     {TW_COS, 0, 3},
+     {TW_SIN, 0, 3},
+     {TW_COS, 0, 9},
+     {TW_SIN, 0, 9},
+     {TW_COS, 0, 15},
+     {TW_SIN, 0, 15},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 16, "t2_16", twinstr, {156, 68, 40, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t2_16) (planner *p) {
+     X(kdft_dit_register) (p, t2_16, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t2_32.c b/src/fftw3/dft/codelets/standard/t2_32.c
new file mode 100644
index 0000000..b065ecb
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t2_32.c
@@ -0,0 +1,853 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:26 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -twiddle-log3 -n 32 -name t2_32 -include t.h */
+
+/*
+ * This function contains 488 FP additions, 280 FP multiplications,
+ * (or, 376 additions, 168 multiplications, 112 fused multiply/add),
+ * 204 stack variables, and 128 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t2_32.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t2_32.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t2_32.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t2_32(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 8) {
+	  E T1, T7G, Tn, Tp, T3t, T4S, TQ, T3G, T49, T20, T2n, T4y, T1J, T43, T2w;
+	  E T4z, T36, T4Z, TK, T8b, T40, T6l, T3U, T6k, T1h, T3L, T1D, T3V, T1s, T3X;
+	  E T3E, T7E, T3O, T6h, T2k, T6w, T4i, T4x, T3q, T6I, T4O, T4P, T3w, T4T, T4R;
+	  E T4U, Tm, To, TX, T4I, T3a, T3H, T31, T4Y, T3f, T4J, T2G, T4s, T4r, T2B;
+	  E T4q, T4t, T27, T4a, T2M, T4m, T4n, T2P, T4l, T4o, T1U, T44;
+	  T1 = ri[0];
+	  T7G = ii[0];
+	  Tn = ri[WS(ios, 16)];
+	  Tp = ii[WS(ios, 16)];
+	  {
+	       E Tv, Tz, TE, TI, TP, TN, TU, TW, T12, T16, T1k, T1b, T1f, T1l, T24;
+	       E T1z, T1w, T1u, T1q, T1o, T1B, T1X, T1Z, T1T, T1R, T1I, T1G, T26, T2O, T3e;
+	       E T3m, T3o, T3u, T3v, T3c, T30, T2W, T33, T35, T38, T39, T2N, T2r, T2v, T2m;
+	       E T2l, T2i, T2g, T2z, T2A, T2D, T2F, T2L, T2J, T2, Ti, T3, Tc, TF, TC;
+	       E TG, TB, Tu, T1a, T15, Ty, T1t, T1Y, T1W, T1v, TH, T1y, T11, TD, T1A;
+	       E T1e, T4g, T3k, T1n, T1p, T2e, T4M, TM, T1K, T1O, TO, T1L, T1N, Ta, Tb;
+	       E T2t, Tk, T2o, Tf, Tg, T2s, Tj, T2p;
+	       Tv = ri[WS(ios, 8)];
+	       Tz = ii[WS(ios, 8)];
+	       TE = ri[WS(ios, 24)];
+	       TI = ii[WS(ios, 24)];
+	       TP = ii[WS(ios, 4)];
+	       TN = ri[WS(ios, 4)];
+	       TU = ri[WS(ios, 20)];
+	       TW = ii[WS(ios, 20)];
+	       T12 = ri[WS(ios, 28)];
+	       T16 = ii[WS(ios, 28)];
+	       T1k = ri[WS(ios, 2)];
+	       T1b = ri[WS(ios, 12)];
+	       T1f = ii[WS(ios, 12)];
+	       T1l = ii[WS(ios, 2)];
+	       T24 = ri[WS(ios, 22)];
+	       T1z = ri[WS(ios, 26)];
+	       T1w = ii[WS(ios, 10)];
+	       T1u = ri[WS(ios, 10)];
+	       T1q = ii[WS(ios, 18)];
+	       T1o = ri[WS(ios, 18)];
+	       T1B = ii[WS(ios, 26)];
+	       T1X = ri[WS(ios, 6)];
+	       T1Z = ii[WS(ios, 6)];
+	       T1T = ii[WS(ios, 14)];
+	       T1R = ri[WS(ios, 14)];
+	       T1I = ii[WS(ios, 30)];
+	       T1G = ri[WS(ios, 30)];
+	       T26 = ii[WS(ios, 22)];
+	       T2O = ii[WS(ios, 13)];
+	       T3e = ii[WS(ios, 23)];
+	       T3m = ri[WS(ios, 19)];
+	       T3o = ii[WS(ios, 19)];
+	       T3u = ri[WS(ios, 11)];
+	       T3v = ii[WS(ios, 11)];
+	       T3c = ri[WS(ios, 23)];
+	       T30 = ii[WS(ios, 31)];
+	       T2W = ri[WS(ios, 31)];
+	       T33 = ri[WS(ios, 15)];
+	       T35 = ii[WS(ios, 15)];
+	       T38 = ri[WS(ios, 7)];
+	       T39 = ii[WS(ios, 7)];
+	       T2N = ri[WS(ios, 13)];
+	       T2r = ri[WS(ios, 25)];
+	       T2v = ii[WS(ios, 25)];
+	       T2m = ii[WS(ios, 9)];
+	       T2l = ri[WS(ios, 9)];
+	       T2i = ii[WS(ios, 17)];
+	       T2g = ri[WS(ios, 17)];
+	       T2z = ri[WS(ios, 5)];
+	       T2A = ii[WS(ios, 5)];
+	       T2D = ri[WS(ios, 21)];
+	       T2F = ii[WS(ios, 21)];
+	       T2L = ii[WS(ios, 29)];
+	       T2J = ri[WS(ios, 29)];
+	       {
+		    E T2c, T2d, T3i, T3j, T3s, T3r, T4, T7, T5, T8, T6, T9, T14, T1d, Ts;
+		    E T18, T19, T1c, Te, Td, Tt, Tw, T13, TZ, T10, Tx;
+		    T2c = ri[WS(ios, 1)];
+		    T2d = ii[WS(ios, 1)];
+		    T3i = ri[WS(ios, 3)];
+		    T3j = ii[WS(ios, 3)];
+		    T3s = ii[WS(ios, 27)];
+		    T3r = ri[WS(ios, 27)];
+		    T2 = W[6];
+		    Ti = W[7];
+		    T3 = W[4];
+		    Tc = W[5];
+		    T4 = W[2];
+		    T7 = W[3];
+		    T5 = W[0];
+		    T8 = W[1];
+		    T6 = T4 * T5;
+		    T9 = T7 * T8;
+		    T14 = Ti * T5;
+		    T1d = Tc * T4;
+		    Ts = T3 * T5;
+		    T18 = T3 * T4;
+		    T19 = Tc * T7;
+		    T1c = T3 * T7;
+		    Te = T7 * T5;
+		    Td = T4 * T8;
+		    Tt = Tc * T8;
+		    Tw = T3 * T8;
+		    TF = T2 * T7;
+		    T13 = T2 * T8;
+		    TC = Ti * T7;
+		    TG = Ti * T4;
+		    TZ = T2 * T5;
+		    T10 = Ti * T8;
+		    TB = T2 * T4;
+		    Tx = Tc * T5;
+		    Tu = Ts + Tt;
+		    T1a = T18 - T19;
+		    T15 = T13 + T14;
+		    Ty = Tw - Tx;
+		    T1t = Ts - Tt;
+		    T1Y = T1c - T1d;
+		    T1W = T18 + T19;
+		    T1v = Tw + Tx;
+		    TH = TF - TG;
+		    T1y = TZ + T10;
+		    T11 = TZ - T10;
+		    TD = TB + TC;
+		    T1A = T13 - T14;
+		    T1e = T1c + T1d;
+		    T3t = FMA(T2, T3r, Ti * T3s);
+		    T4g = FNMS(T8, T2c, T5 * T2d);
+		    T4S = FNMS(Ti, T3r, T2 * T3s);
+		    T3k = FMA(T4, T3i, T7 * T3j);
+		    T1n = FMA(T2, T3, Ti * Tc);
+		    T1p = FNMS(Ti, T3, T2 * Tc);
+		    T2e = FMA(T5, T2c, T8 * T2d);
+		    T4M = FNMS(T7, T3i, T4 * T3j);
+		    TM = T6 - T9;
+		    T1K = T3 * TM;
+		    T1O = Tc * TM;
+		    TO = Td + Te;
+		    T1L = Tc * TO;
+		    T1N = T3 * TO;
+		    Ta = T6 + T9;
+		    Tb = T3 * Ta;
+		    T2t = Ti * Ta;
+		    Tk = Tc * Ta;
+		    T2o = T2 * Ta;
+		    Tf = Td - Te;
+		    Tg = Tc * Tf;
+		    T2s = T2 * Tf;
+		    Tj = T3 * Tf;
+		    T2p = Ti * Tf;
+	       }
+	       TQ = FMA(TM, TN, TO * TP);
+	       T3G = FNMS(TO, TN, TM * TP);
+	       T49 = FMA(T1Y, T1X, T1W * T1Z);
+	       T20 = FNMS(T1Y, T1Z, T1W * T1X);
+	       T2n = FMA(T3, T2l, Tc * T2m);
+	       T4y = FNMS(Tc, T2l, T3 * T2m);
+	       {
+		    E T1F, T1H, TA, TJ;
+		    T1F = TB - TC;
+		    T1H = TF + TG;
+		    T1J = FMA(T1F, T1G, T1H * T1I);
+		    T43 = FNMS(T1H, T1G, T1F * T1I);
+		    {
+			 E T2q, T2u, T32, T34;
+			 T2q = T2o - T2p;
+			 T2u = T2s + T2t;
+			 T2w = FMA(T2q, T2r, T2u * T2v);
+			 T4z = FNMS(T2u, T2r, T2q * T2v);
+			 T32 = FMA(T2, T1a, Ti * T1e);
+			 T34 = FNMS(Ti, T1a, T2 * T1e);
+			 T36 = FNMS(T34, T35, T32 * T33);
+			 T4Z = FMA(T34, T33, T32 * T35);
+		    }
+		    TA = FNMS(Ty, Tz, Tu * Tv);
+		    TJ = FNMS(TH, TI, TD * TE);
+		    TK = TA + TJ;
+		    T8b = TA - TJ;
+		    {
+			 E T3Y, T3Z, T3S, T3T;
+			 T3Y = FNMS(T1v, T1u, T1t * T1w);
+			 T3Z = FMA(T1A, T1z, T1y * T1B);
+			 T40 = T3Y - T3Z;
+			 T6l = T3Y + T3Z;
+			 T3S = FMA(Tf, T1k, Ta * T1l);
+			 T3T = FMA(T1p, T1o, T1n * T1q);
+			 T3U = T3S - T3T;
+			 T6k = T3S + T3T;
+		    }
+	       }
+	       {
+		    E T17, T1g, Th, Tl;
+		    T17 = FMA(T11, T12, T15 * T16);
+		    T1g = FMA(T1a, T1b, T1e * T1f);
+		    T1h = T17 + T1g;
+		    T3L = T17 - T1g;
+		    {
+			 E T1x, T1C, T1m, T1r;
+			 T1x = FMA(T1t, T1u, T1v * T1w);
+			 T1C = FNMS(T1A, T1B, T1y * T1z);
+			 T1D = T1x + T1C;
+			 T3V = T1x - T1C;
+			 T1m = FNMS(Tf, T1l, Ta * T1k);
+			 T1r = FNMS(T1p, T1q, T1n * T1o);
+			 T1s = T1m + T1r;
+			 T3X = T1m - T1r;
+		    }
+		    {
+			 E T3C, T3D, T3M, T3N;
+			 T3C = FMA(Ty, Tv, Tu * Tz);
+			 T3D = FMA(TH, TE, TD * TI);
+			 T3E = T3C - T3D;
+			 T7E = T3C + T3D;
+			 T3M = FNMS(T15, T12, T11 * T16);
+			 T3N = FNMS(T1e, T1b, T1a * T1f);
+			 T3O = T3M - T3N;
+			 T6h = T3M + T3N;
+			 {
+			      E T2j, T4h, T2f, T2h;
+			      T2f = FMA(T2, T1t, Ti * T1v);
+			      T2h = FNMS(Ti, T1t, T2 * T1v);
+			      T2j = FNMS(T2h, T2i, T2f * T2g);
+			      T4h = FMA(T2h, T2g, T2f * T2i);
+			      T2k = T2e + T2j;
+			      T6w = T4g + T4h;
+			      T4i = T4g - T4h;
+			      T4x = T2e - T2j;
+			 }
+		    }
+		    {
+			 E T3p, T4N, T3l, T3n;
+			 T3l = FNMS(Ti, Ty, T2 * Tu);
+			 T3n = FMA(T2, Ty, Ti * Tu);
+			 T3p = FMA(T3l, T3m, T3n * T3o);
+			 T4N = FNMS(T3n, T3m, T3l * T3o);
+			 T3q = T3k + T3p;
+			 T6I = T4M + T4N;
+			 T4O = T4M - T4N;
+			 T4P = T3k - T3p;
+		    }
+		    Th = Tb + Tg;
+		    Tl = Tj - Tk;
+		    T3w = FNMS(Tl, T3v, Th * T3u);
+		    T4T = FMA(Tl, T3u, Th * T3v);
+		    T4R = T3t - T3w;
+		    T4U = T4S - T4T;
+		    Tm = FNMS(Ti, Tl, T2 * Th);
+		    To = FMA(T2, Tl, Ti * Th);
+		    {
+			 E TR, TS, TT, TV;
+			 TR = Tb - Tg;
+			 TS = Tj + Tk;
+			 TT = FMA(T2, TR, Ti * TS);
+			 TV = FNMS(Ti, TR, T2 * TS);
+			 TX = FNMS(TV, TW, TT * TU);
+			 T4I = FNMS(TS, T38, TR * T39);
+			 T3a = FMA(TR, T38, TS * T39);
+			 T3H = FMA(TV, TU, TT * TW);
+		    }
+		    {
+			 E T2V, T3b, T2Z, T3d;
+			 {
+			      E T2T, T2U, T2X, T2Y;
+			      T2T = T2 * TM;
+			      T2U = Ti * TO;
+			      T2V = T2T - T2U;
+			      T3b = T2T + T2U;
+			      T2X = T2 * TO;
+			      T2Y = Ti * TM;
+			      T2Z = T2X + T2Y;
+			      T3d = T2X - T2Y;
+			 }
+			 T31 = FMA(T2V, T2W, T2Z * T30);
+			 T4Y = FNMS(T2Z, T2W, T2V * T30);
+			 T3f = FNMS(T3d, T3e, T3b * T3c);
+			 T4J = FMA(T3d, T3c, T3b * T3e);
+		    }
+		    {
+			 E T23, T25, T1Q, T1S;
+			 {
+			      E T2C, T2E, T21, T22;
+			      T2C = FNMS(Ti, T1Y, T2 * T1W);
+			      T2E = FMA(T2, T1Y, Ti * T1W);
+			      T2G = FMA(T2C, T2D, T2E * T2F);
+			      T4s = FNMS(T2E, T2D, T2C * T2F);
+			      T21 = T1K + T1L;
+			      T22 = T1N - T1O;
+			      T23 = FNMS(Ti, T22, T2 * T21);
+			      T4r = FMA(T22, T2z, T21 * T2A);
+			      T25 = FMA(T2, T22, Ti * T21);
+			      T2B = FNMS(T22, T2A, T21 * T2z);
+			 }
+			 T4q = T2B - T2G;
+			 T4t = T4r - T4s;
+			 T27 = FMA(T23, T24, T25 * T26);
+			 T4a = FNMS(T25, T24, T23 * T26);
+			 {
+			      E T2I, T2K, T1M, T1P;
+			      T2I = T2o + T2p;
+			      T2K = T2s - T2t;
+			      T2M = FNMS(T2K, T2L, T2I * T2J);
+			      T4m = FMA(T2K, T2J, T2I * T2L);
+			      T1M = T1K - T1L;
+			      T1P = T1N + T1O;
+			      T1Q = FMA(T2, T1M, Ti * T1P);
+			      T4n = FNMS(T1P, T2N, T1M * T2O);
+			      T1S = FNMS(Ti, T1M, T2 * T1P);
+			      T2P = FMA(T1M, T2N, T1P * T2O);
+			 }
+			 T4l = T2M - T2P;
+			 T4o = T4m - T4n;
+			 T1U = FNMS(T1S, T1T, T1Q * T1R);
+			 T44 = FMA(T1S, T1R, T1Q * T1T);
+		    }
+	       }
+	  }
+	  {
+	       E T1i, T7V, T6i, T7D, T42, T5e, T5A, T60, T6o, T6Y, TL, T6f, T3F, T5t, T7I;
+	       E T8q, T7W, T8c, T3Q, T8p, T5w, T89, T4d, T61, T5f, T5D, T2a, T6t, T7O, T7C;
+	       E T7g, T6Z, T4w, T64, T65, T4F, T5i, T5I, T5L, T5j, T2S, T7l, T7y, T6A, T6F;
+	       E T73, T7i, T72, T4X, T67, T68, T56, T5l, T5P, T5S, T5m, T3z, T7q, T7z, T6L;
+	       E T6Q, T76, T7n, T75;
+	       {
+		    E TY, T6g, T3W, T41;
+		    TY = TQ + TX;
+		    T1i = TY + T1h;
+		    T7V = T1h - TY;
+		    T6g = T3G + T3H;
+		    T6i = T6g - T6h;
+		    T7D = T6g + T6h;
+		    T3W = T3U + T3V;
+		    T41 = T3X - T40;
+		    T42 = FNMS(KP923879532, T41, KP382683432 * T3W);
+		    T5e = FMA(KP923879532, T3W, KP382683432 * T41);
+	       }
+	       {
+		    E T5y, T5z, T6m, T6n;
+		    T5y = T3U - T3V;
+		    T5z = T3X + T40;
+		    T5A = FNMS(KP382683432, T5z, KP923879532 * T5y);
+		    T60 = FMA(KP382683432, T5y, KP923879532 * T5z);
+		    T6m = T6k - T6l;
+		    T6n = T1s - T1D;
+		    T6o = T6m - T6n;
+		    T6Y = T6n + T6m;
+	       }
+	       {
+		    E Tr, T3B, Tq, T7H, T8a, T7F;
+		    Tq = FMA(Tm, Tn, To * Tp);
+		    Tr = T1 + Tq;
+		    T3B = T1 - Tq;
+		    TL = Tr + TK;
+		    T6f = Tr - TK;
+		    T3F = T3B - T3E;
+		    T5t = T3B + T3E;
+		    T7F = FNMS(To, Tn, Tm * Tp);
+		    T7H = T7F + T7G;
+		    T8a = T7G - T7F;
+		    T7I = T7E + T7H;
+		    T8q = T8b + T8a;
+		    T7W = T7H - T7E;
+		    T8c = T8a - T8b;
+	       }
+	       {
+		    E T3P, T5v, T3K, T5u, T3I, T3J;
+		    T3P = T3L + T3O;
+		    T5v = T3L - T3O;
+		    T3I = T3G - T3H;
+		    T3J = TQ - TX;
+		    T3K = T3I - T3J;
+		    T5u = T3J + T3I;
+		    T3Q = KP707106781 * (T3K - T3P);
+		    T8p = KP707106781 * (T5v - T5u);
+		    T5w = KP707106781 * (T5u + T5v);
+		    T89 = KP707106781 * (T3K + T3P);
+	       }
+	       {
+		    E T47, T5B, T4c, T5C;
+		    {
+			 E T45, T46, T48, T4b;
+			 T45 = T43 - T44;
+			 T46 = T20 - T27;
+			 T47 = T45 + T46;
+			 T5B = T45 - T46;
+			 T48 = T1J - T1U;
+			 T4b = T49 - T4a;
+			 T4c = T48 - T4b;
+			 T5C = T48 + T4b;
+		    }
+		    T4d = FMA(KP382683432, T47, KP923879532 * T4c);
+		    T61 = FNMS(KP382683432, T5B, KP923879532 * T5C);
+		    T5f = FNMS(KP923879532, T47, KP382683432 * T4c);
+		    T5D = FMA(KP923879532, T5B, KP382683432 * T5C);
+	       }
+	       {
+		    E T1E, T7e, T29, T6p, T6s, T7f;
+		    T1E = T1s + T1D;
+		    T7e = T6k + T6l;
+		    {
+			 E T1V, T28, T6q, T6r;
+			 T1V = T1J + T1U;
+			 T28 = T20 + T27;
+			 T29 = T1V + T28;
+			 T6p = T1V - T28;
+			 T6q = T43 + T44;
+			 T6r = T49 + T4a;
+			 T6s = T6q - T6r;
+			 T7f = T6q + T6r;
+		    }
+		    T2a = T1E + T29;
+		    T6t = T6p + T6s;
+		    T7O = T29 - T1E;
+		    T7C = T7e + T7f;
+		    T7g = T7e - T7f;
+		    T6Z = T6p - T6s;
+	       }
+	       {
+		    E T4k, T5J, T4B, T5G, T4v, T5H, T4E, T5K, T4j, T4A;
+		    T4j = T2n - T2w;
+		    T4k = T4i + T4j;
+		    T5J = T4i - T4j;
+		    T4A = T4y - T4z;
+		    T4B = T4x - T4A;
+		    T5G = T4x + T4A;
+		    {
+			 E T4p, T4u, T4C, T4D;
+			 T4p = T4l - T4o;
+			 T4u = T4q + T4t;
+			 T4v = KP707106781 * (T4p - T4u);
+			 T5H = KP707106781 * (T4u + T4p);
+			 T4C = T4t - T4q;
+			 T4D = T4l + T4o;
+			 T4E = KP707106781 * (T4C - T4D);
+			 T5K = KP707106781 * (T4C + T4D);
+		    }
+		    T4w = T4k - T4v;
+		    T64 = T5G + T5H;
+		    T65 = T5J + T5K;
+		    T4F = T4B - T4E;
+		    T5i = T4k + T4v;
+		    T5I = T5G - T5H;
+		    T5L = T5J - T5K;
+		    T5j = T4B + T4E;
+	       }
+	       {
+		    E T2y, T6B, T6y, T7j, T2R, T6z, T6E, T7k, T2x, T6x;
+		    T2x = T2n + T2w;
+		    T2y = T2k + T2x;
+		    T6B = T2k - T2x;
+		    T6x = T4y + T4z;
+		    T6y = T6w - T6x;
+		    T7j = T6w + T6x;
+		    {
+			 E T2H, T2Q, T6C, T6D;
+			 T2H = T2B + T2G;
+			 T2Q = T2M + T2P;
+			 T2R = T2H + T2Q;
+			 T6z = T2Q - T2H;
+			 T6C = T4r + T4s;
+			 T6D = T4m + T4n;
+			 T6E = T6C - T6D;
+			 T7k = T6C + T6D;
+		    }
+		    T2S = T2y + T2R;
+		    T7l = T7j - T7k;
+		    T7y = T7j + T7k;
+		    T6A = T6y - T6z;
+		    T6F = T6B - T6E;
+		    T73 = T6B + T6E;
+		    T7i = T2y - T2R;
+		    T72 = T6y + T6z;
+	       }
+	       {
+		    E T4L, T5N, T55, T5O, T4W, T5R, T52, T5Q;
+		    {
+			 E T4H, T4K, T53, T54;
+			 T4H = T31 - T36;
+			 T4K = T4I - T4J;
+			 T4L = T4H - T4K;
+			 T5N = T4H + T4K;
+			 T53 = T4R - T4U;
+			 T54 = T4P + T4O;
+			 T55 = KP707106781 * (T53 - T54);
+			 T5O = KP707106781 * (T54 + T53);
+		    }
+		    {
+			 E T4Q, T4V, T50, T51;
+			 T4Q = T4O - T4P;
+			 T4V = T4R + T4U;
+			 T4W = KP707106781 * (T4Q - T4V);
+			 T5R = KP707106781 * (T4Q + T4V);
+			 T50 = T4Y - T4Z;
+			 T51 = T3a - T3f;
+			 T52 = T50 + T51;
+			 T5Q = T50 - T51;
+		    }
+		    T4X = T4L - T4W;
+		    T67 = T5N + T5O;
+		    T68 = T5Q + T5R;
+		    T56 = T52 - T55;
+		    T5l = T4L + T4W;
+		    T5P = T5N - T5O;
+		    T5S = T5Q - T5R;
+		    T5m = T52 + T55;
+	       }
+	       {
+		    E T3y, T6P, T6K, T7p, T3h, T6H, T6O, T7o, T3x, T6J;
+		    T3x = T3t + T3w;
+		    T3y = T3q + T3x;
+		    T6P = T3x - T3q;
+		    T6J = T4S + T4T;
+		    T6K = T6I - T6J;
+		    T7p = T6I + T6J;
+		    {
+			 E T37, T3g, T6M, T6N;
+			 T37 = T31 + T36;
+			 T3g = T3a + T3f;
+			 T3h = T37 + T3g;
+			 T6H = T37 - T3g;
+			 T6M = T4Y + T4Z;
+			 T6N = T4I + T4J;
+			 T6O = T6M - T6N;
+			 T7o = T6M + T6N;
+		    }
+		    T3z = T3h + T3y;
+		    T7q = T7o - T7p;
+		    T7z = T7o + T7p;
+		    T6L = T6H - T6K;
+		    T6Q = T6O - T6P;
+		    T76 = T6O + T6P;
+		    T7n = T3h - T3y;
+		    T75 = T6H + T6K;
+	       }
+	       {
+		    E T3A, T7A, T2b, T7x, T1j;
+		    T3A = T2S + T3z;
+		    T7A = T7y - T7z;
+		    T1j = TL + T1i;
+		    T2b = T1j + T2a;
+		    T7x = T1j - T2a;
+		    ri[WS(ios, 16)] = T2b - T3A;
+		    ri[WS(ios, 8)] = T7x + T7A;
+		    ri[0] = T2b + T3A;
+		    ri[WS(ios, 24)] = T7x - T7A;
+	       }
+	       {
+		    E T7B, T7L, T7K, T7M, T7J;
+		    T7B = T7y + T7z;
+		    T7L = T3z - T2S;
+		    T7J = T7D + T7I;
+		    T7K = T7C + T7J;
+		    T7M = T7J - T7C;
+		    ii[0] = T7B + T7K;
+		    ii[WS(ios, 24)] = T7M - T7L;
+		    ii[WS(ios, 16)] = T7K - T7B;
+		    ii[WS(ios, 8)] = T7L + T7M;
+	       }
+	       {
+		    E T7h, T7t, T7Q, T7S, T7s, T7R, T7w, T7N, T7d, T7P;
+		    T7d = TL - T1i;
+		    T7h = T7d + T7g;
+		    T7t = T7d - T7g;
+		    T7P = T7I - T7D;
+		    T7Q = T7O + T7P;
+		    T7S = T7P - T7O;
+		    {
+			 E T7m, T7r, T7u, T7v;
+			 T7m = T7i + T7l;
+			 T7r = T7n - T7q;
+			 T7s = KP707106781 * (T7m + T7r);
+			 T7R = KP707106781 * (T7r - T7m);
+			 T7u = T7l - T7i;
+			 T7v = T7n + T7q;
+			 T7w = KP707106781 * (T7u - T7v);
+			 T7N = KP707106781 * (T7u + T7v);
+		    }
+		    ri[WS(ios, 20)] = T7h - T7s;
+		    ii[WS(ios, 20)] = T7Q - T7N;
+		    ri[WS(ios, 4)] = T7h + T7s;
+		    ii[WS(ios, 4)] = T7N + T7Q;
+		    ri[WS(ios, 28)] = T7t - T7w;
+		    ii[WS(ios, 28)] = T7S - T7R;
+		    ri[WS(ios, 12)] = T7t + T7w;
+		    ii[WS(ios, 12)] = T7R + T7S;
+	       }
+	       {
+		    E T71, T79, T7Y, T80, T78, T7Z, T7c, T7T;
+		    {
+			 E T6X, T70, T7U, T7X;
+			 T6X = T6f + T6i;
+			 T70 = KP707106781 * (T6Y + T6Z);
+			 T71 = T6X + T70;
+			 T79 = T6X - T70;
+			 T7U = KP707106781 * (T6o + T6t);
+			 T7X = T7V + T7W;
+			 T7Y = T7U + T7X;
+			 T80 = T7X - T7U;
+		    }
+		    {
+			 E T74, T77, T7a, T7b;
+			 T74 = FMA(KP382683432, T72, KP923879532 * T73);
+			 T77 = FNMS(KP382683432, T76, KP923879532 * T75);
+			 T78 = T74 + T77;
+			 T7Z = T77 - T74;
+			 T7a = FNMS(KP382683432, T73, KP923879532 * T72);
+			 T7b = FMA(KP923879532, T76, KP382683432 * T75);
+			 T7c = T7a - T7b;
+			 T7T = T7a + T7b;
+		    }
+		    ri[WS(ios, 18)] = T71 - T78;
+		    ii[WS(ios, 18)] = T7Y - T7T;
+		    ri[WS(ios, 2)] = T71 + T78;
+		    ii[WS(ios, 2)] = T7T + T7Y;
+		    ri[WS(ios, 26)] = T79 - T7c;
+		    ii[WS(ios, 26)] = T80 - T7Z;
+		    ri[WS(ios, 10)] = T79 + T7c;
+		    ii[WS(ios, 10)] = T7Z + T80;
+	       }
+	       {
+		    E T4f, T59, T8y, T8A, T58, T8z, T5c, T8v;
+		    {
+			 E T3R, T4e, T8w, T8x;
+			 T3R = T3F - T3Q;
+			 T4e = T42 - T4d;
+			 T4f = T3R + T4e;
+			 T59 = T3R - T4e;
+			 T8w = T5f - T5e;
+			 T8x = T8q - T8p;
+			 T8y = T8w + T8x;
+			 T8A = T8x - T8w;
+		    }
+		    {
+			 E T4G, T57, T5a, T5b;
+			 T4G = FMA(KP980785280, T4w, KP195090322 * T4F);
+			 T57 = FNMS(KP980785280, T56, KP195090322 * T4X);
+			 T58 = T4G + T57;
+			 T8z = T57 - T4G;
+			 T5a = FNMS(KP980785280, T4F, KP195090322 * T4w);
+			 T5b = FMA(KP195090322, T56, KP980785280 * T4X);
+			 T5c = T5a - T5b;
+			 T8v = T5a + T5b;
+		    }
+		    ri[WS(ios, 23)] = T4f - T58;
+		    ii[WS(ios, 23)] = T8y - T8v;
+		    ri[WS(ios, 7)] = T4f + T58;
+		    ii[WS(ios, 7)] = T8v + T8y;
+		    ri[WS(ios, 31)] = T59 - T5c;
+		    ii[WS(ios, 31)] = T8A - T8z;
+		    ri[WS(ios, 15)] = T59 + T5c;
+		    ii[WS(ios, 15)] = T8z + T8A;
+	       }
+	       {
+		    E T5F, T5V, T8k, T8m, T5U, T8l, T5Y, T8h;
+		    {
+			 E T5x, T5E, T8i, T8j;
+			 T5x = T5t - T5w;
+			 T5E = T5A - T5D;
+			 T5F = T5x + T5E;
+			 T5V = T5x - T5E;
+			 T8i = T61 - T60;
+			 T8j = T8c - T89;
+			 T8k = T8i + T8j;
+			 T8m = T8j - T8i;
+		    }
+		    {
+			 E T5M, T5T, T5W, T5X;
+			 T5M = FMA(KP555570233, T5I, KP831469612 * T5L);
+			 T5T = FNMS(KP831469612, T5S, KP555570233 * T5P);
+			 T5U = T5M + T5T;
+			 T8l = T5T - T5M;
+			 T5W = FNMS(KP831469612, T5I, KP555570233 * T5L);
+			 T5X = FMA(KP831469612, T5P, KP555570233 * T5S);
+			 T5Y = T5W - T5X;
+			 T8h = T5W + T5X;
+		    }
+		    ri[WS(ios, 21)] = T5F - T5U;
+		    ii[WS(ios, 21)] = T8k - T8h;
+		    ri[WS(ios, 5)] = T5F + T5U;
+		    ii[WS(ios, 5)] = T8h + T8k;
+		    ri[WS(ios, 29)] = T5V - T5Y;
+		    ii[WS(ios, 29)] = T8m - T8l;
+		    ri[WS(ios, 13)] = T5V + T5Y;
+		    ii[WS(ios, 13)] = T8l + T8m;
+	       }
+	       {
+		    E T6v, T6T, T84, T86, T6S, T85, T6W, T81;
+		    {
+			 E T6j, T6u, T82, T83;
+			 T6j = T6f - T6i;
+			 T6u = KP707106781 * (T6o - T6t);
+			 T6v = T6j + T6u;
+			 T6T = T6j - T6u;
+			 T82 = KP707106781 * (T6Z - T6Y);
+			 T83 = T7W - T7V;
+			 T84 = T82 + T83;
+			 T86 = T83 - T82;
+		    }
+		    {
+			 E T6G, T6R, T6U, T6V;
+			 T6G = FMA(KP923879532, T6A, KP382683432 * T6F);
+			 T6R = FNMS(KP923879532, T6Q, KP382683432 * T6L);
+			 T6S = T6G + T6R;
+			 T85 = T6R - T6G;
+			 T6U = FNMS(KP923879532, T6F, KP382683432 * T6A);
+			 T6V = FMA(KP382683432, T6Q, KP923879532 * T6L);
+			 T6W = T6U - T6V;
+			 T81 = T6U + T6V;
+		    }
+		    ri[WS(ios, 22)] = T6v - T6S;
+		    ii[WS(ios, 22)] = T84 - T81;
+		    ri[WS(ios, 6)] = T6v + T6S;
+		    ii[WS(ios, 6)] = T81 + T84;
+		    ri[WS(ios, 30)] = T6T - T6W;
+		    ii[WS(ios, 30)] = T86 - T85;
+		    ri[WS(ios, 14)] = T6T + T6W;
+		    ii[WS(ios, 14)] = T85 + T86;
+	       }
+	       {
+		    E T5h, T5p, T8s, T8u, T5o, T8t, T5s, T8n;
+		    {
+			 E T5d, T5g, T8o, T8r;
+			 T5d = T3F + T3Q;
+			 T5g = T5e + T5f;
+			 T5h = T5d + T5g;
+			 T5p = T5d - T5g;
+			 T8o = T42 + T4d;
+			 T8r = T8p + T8q;
+			 T8s = T8o + T8r;
+			 T8u = T8r - T8o;
+		    }
+		    {
+			 E T5k, T5n, T5q, T5r;
+			 T5k = FMA(KP555570233, T5i, KP831469612 * T5j);
+			 T5n = FNMS(KP555570233, T5m, KP831469612 * T5l);
+			 T5o = T5k + T5n;
+			 T8t = T5n - T5k;
+			 T5q = FNMS(KP555570233, T5j, KP831469612 * T5i);
+			 T5r = FMA(KP831469612, T5m, KP555570233 * T5l);
+			 T5s = T5q - T5r;
+			 T8n = T5q + T5r;
+		    }
+		    ri[WS(ios, 19)] = T5h - T5o;
+		    ii[WS(ios, 19)] = T8s - T8n;
+		    ri[WS(ios, 3)] = T5h + T5o;
+		    ii[WS(ios, 3)] = T8n + T8s;
+		    ri[WS(ios, 27)] = T5p - T5s;
+		    ii[WS(ios, 27)] = T8u - T8t;
+		    ri[WS(ios, 11)] = T5p + T5s;
+		    ii[WS(ios, 11)] = T8t + T8u;
+	       }
+	       {
+		    E T63, T6b, T8e, T8g, T6a, T8f, T6e, T87;
+		    {
+			 E T5Z, T62, T88, T8d;
+			 T5Z = T5t + T5w;
+			 T62 = T60 + T61;
+			 T63 = T5Z + T62;
+			 T6b = T5Z - T62;
+			 T88 = T5A + T5D;
+			 T8d = T89 + T8c;
+			 T8e = T88 + T8d;
+			 T8g = T8d - T88;
+		    }
+		    {
+			 E T66, T69, T6c, T6d;
+			 T66 = FMA(KP980785280, T64, KP195090322 * T65);
+			 T69 = FNMS(KP195090322, T68, KP980785280 * T67);
+			 T6a = T66 + T69;
+			 T8f = T69 - T66;
+			 T6c = FNMS(KP195090322, T64, KP980785280 * T65);
+			 T6d = FMA(KP195090322, T67, KP980785280 * T68);
+			 T6e = T6c - T6d;
+			 T87 = T6c + T6d;
+		    }
+		    ri[WS(ios, 17)] = T63 - T6a;
+		    ii[WS(ios, 17)] = T8e - T87;
+		    ri[WS(ios, 1)] = T63 + T6a;
+		    ii[WS(ios, 1)] = T87 + T8e;
+		    ri[WS(ios, 25)] = T6b - T6e;
+		    ii[WS(ios, 25)] = T8g - T8f;
+		    ri[WS(ios, 9)] = T6b + T6e;
+		    ii[WS(ios, 9)] = T8f + T8g;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_COS, 0, 1},
+     {TW_SIN, 0, 1},
+     {TW_COS, 0, 3},
+     {TW_SIN, 0, 3},
+     {TW_COS, 0, 9},
+     {TW_SIN, 0, 9},
+     {TW_COS, 0, 27},
+     {TW_SIN, 0, 27},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 32, "t2_32", twinstr, {376, 168, 112, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t2_32) (planner *p) {
+     X(kdft_dit_register) (p, t2_32, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t2_4.c b/src/fftw3/dft/codelets/standard/t2_4.c
new file mode 100644
index 0000000..ced1a63
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t2_4.c
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:08 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -twiddle-log3 -n 4 -name t2_4 -include t.h */
+
+/*
+ * This function contains 24 FP additions, 16 FP multiplications,
+ * (or, 16 additions, 8 multiplications, 8 fused multiply/add),
+ * 21 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t2_4.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t2_4.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t2_4.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t2_4(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 4) {
+	  E T1, Tp, Ta, Te, To, Tl, Tk, Th;
+	  T1 = ri[0];
+	  Tp = ii[0];
+	  {
+	       E T7, T9, Tc, Td, Tg, Tf, T2, T4, T3, T5, T6, T8;
+	       T7 = ri[WS(ios, 2)];
+	       T9 = ii[WS(ios, 2)];
+	       Tc = ri[WS(ios, 1)];
+	       Td = ii[WS(ios, 1)];
+	       Tg = ii[WS(ios, 3)];
+	       Tf = ri[WS(ios, 3)];
+	       T2 = W[2];
+	       T4 = W[3];
+	       T3 = W[0];
+	       T5 = W[1];
+	       T6 = FMA(T2, T3, T4 * T5);
+	       T8 = FNMS(T4, T3, T2 * T5);
+	       Ta = FNMS(T8, T9, T6 * T7);
+	       Te = FMA(T3, Tc, T5 * Td);
+	       To = FMA(T8, T7, T6 * T9);
+	       Tl = FNMS(T4, Tf, T2 * Tg);
+	       Tk = FNMS(T5, Tc, T3 * Td);
+	       Th = FMA(T2, Tf, T4 * Tg);
+	  }
+	  {
+	       E Tb, Ti, Tn, Tq;
+	       Tb = T1 + Ta;
+	       Ti = Te + Th;
+	       ri[WS(ios, 2)] = Tb - Ti;
+	       ri[0] = Tb + Ti;
+	       Tn = Tk + Tl;
+	       Tq = To + Tp;
+	       ii[0] = Tn + Tq;
+	       ii[WS(ios, 2)] = Tq - Tn;
+	  }
+	  {
+	       E Tj, Tm, Tr, Ts;
+	       Tj = T1 - Ta;
+	       Tm = Tk - Tl;
+	       ri[WS(ios, 3)] = Tj - Tm;
+	       ri[WS(ios, 1)] = Tj + Tm;
+	       Tr = Tp - To;
+	       Ts = Te - Th;
+	       ii[WS(ios, 1)] = Tr - Ts;
+	       ii[WS(ios, 3)] = Ts + Tr;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_COS, 0, 1},
+     {TW_SIN, 0, 1},
+     {TW_COS, 0, 3},
+     {TW_SIN, 0, 3},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 4, "t2_4", twinstr, {16, 8, 8, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t2_4) (planner *p) {
+     X(kdft_dit_register) (p, t2_4, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t2_64.c b/src/fftw3/dft/codelets/standard/t2_64.c
new file mode 100644
index 0000000..6fc7efd
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t2_64.c
@@ -0,0 +1,1906 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:32 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -twiddle-log3 -n 64 -name t2_64 -include t.h */
+
+/*
+ * This function contains 1154 FP additions, 660 FP multiplications,
+ * (or, 880 additions, 386 multiplications, 274 fused multiply/add),
+ * 382 stack variables, and 256 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t2_64.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t2_64.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t2_64.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t2_64(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP290284677, +0.290284677254462367636192375817395274691476278);
+     DK(KP956940335, +0.956940335732208864935797886980269969482849206);
+     DK(KP881921264, +0.881921264348355029712756863660388349508442621);
+     DK(KP471396736, +0.471396736825997648556387625905254377657460319);
+     DK(KP098017140, +0.098017140329560601994195563888641845861136673);
+     DK(KP995184726, +0.995184726672196886244836953109479921575474869);
+     DK(KP773010453, +0.773010453362736960810906609758469800971041293);
+     DK(KP634393284, +0.634393284163645498215171613225493370675687095);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 10) {
+	  E T1, Ti1, Tp, Tt, TH, TL, T6a, T6c, T4J, T4H, T1g, T91, T7W, T7m, T2O;
+	  E T4j, T7P, T4P, T8y, T2w, T8t, T2Z, T8e, T48, T1z, T7s, T1I, T7t, T8p, Ten;
+	  E T1Y, T7D, T2t, T7O, T7L, Te6, T3N, T8E, T7A, Te0, T4C, TeA, T8S, T9v, T65;
+	  E Tfi, T9J, Taq, T6K, Tf6, Ta2, Ta5, T73, Tfc, Tad, Tag, T3z, T83, T3u, T82;
+	  E T81, T84, T15, T9K, T68, T7j, T43, T9w, T4F, T8G, T5l, TeL, T9k, T9n, T6o;
+	  E Tf2, T9Q, T9R, T6z, Tf3, T9T, T9W, To, Ts, T4o, T8u, T4U, T92, T5a, TeT;
+	  E T8V, T8Y, T5G, TeG, T97, T9e, T27, T7X, T2T, T7E, T7b, Tai, T6T, Ta3, Tf7;
+	  E Ta8, T7Q, T2H, T2c, T76, Tah, T7F, T4d, T8z, TG, TK, T69, T6b, T3b, T87;
+	  E T5u, T9l, TeM, T9q, T88, T89, T3o, T86, T5P, T9f, TeH, T9a, T34, T8f, T1r;
+	  E T7n, T3S, T8F, T4G, T4I;
+	  T1 = ri[0];
+	  Ti1 = ii[0];
+	  Tp = ri[WS(ios, 32)];
+	  Tt = ii[WS(ios, 32)];
+	  TH = ri[WS(ios, 16)];
+	  TL = ii[WS(ios, 16)];
+	  T6a = ri[WS(ios, 47)];
+	  T6c = ii[WS(ios, 47)];
+	  T4J = ii[WS(ios, 49)];
+	  T4H = ri[WS(ios, 49)];
+	  {
+	       E T12, T14, T1b, T1f, T1q, T1m, T1w, T1y, T1D, T1H, T1S, T1M, T1N, T1W, T2M;
+	       E T2g, T2b, T29, T26, T22, T2i, T2E, T2G, T2v, T2u, T2r, T2n, T2N, T3a, T38;
+	       E T3l, T3n, T3r, T3t, T33, T31, T2Y, T2W, T4g, T2S, T2Q, T3w, T3y, T3E, T3G;
+	       E T3P, T3J, T3L, T3R, T4a, T4c, T47, T46, T42, T40, T4i, T6P, T6R, T6M, T6L;
+	       E T6I, T6G, T6W, T6Y, T74, T75, T5I, T78, T7a, T6x, T6v, T6s, T6q, T6h, T6m;
+	       E T6k, T6g, T5N, T5L, T5Z, T63, T66, T67, T5H, T54, T4D, T4A, T4y, T4n, T4l;
+	       E T4E, T4X, T4Z, T4T, T4R, T4O, T4N, T58, T5s, T5q, T5x, T5z, T5C, T5E, T5n;
+	       E T5m, T5j, T5h, T5d, T5e, Ta, Ty, Tf, Tw, T2, Tj, T3, Tc, T1E, T1B;
+	       E T1F, T1A, T1R, T3x, T2m, T3K, T61, T1V, T60, T3I, T51, T52, T2V, T56, T5X;
+	       E T3v, T55, T2X, T2q, T5W, T4w, T6E, Ta0, T8Q, Tac, T72, Tb, Tg, Th, T3e;
+	       E T3f, T3h, T1a, T2x, T2B, TU, TV, TY, T1e, T2y, T2A, TC, TD, T1u, Tk;
+	       E Tl, Tm, T39, T3U, T3W, T37, T3T, T3X, TQ, TR, TZ, T3c, T3d, T3i, Tx;
+	       E Tz, T1t, TN, TX, T2f, T5V, Tao, T2h, T3D, T4f, T4h, T3F, T3q, T3s;
+	       T12 = ri[WS(ios, 48)];
+	       T14 = ii[WS(ios, 48)];
+	       T1b = ri[WS(ios, 8)];
+	       T1f = ii[WS(ios, 8)];
+	       T1q = ii[WS(ios, 40)];
+	       T1m = ri[WS(ios, 40)];
+	       T1w = ri[WS(ios, 56)];
+	       T1y = ii[WS(ios, 56)];
+	       T1D = ri[WS(ios, 24)];
+	       T1H = ii[WS(ios, 24)];
+	       T1S = ri[WS(ios, 36)];
+	       T1M = ri[WS(ios, 4)];
+	       T1N = ii[WS(ios, 4)];
+	       T1W = ii[WS(ios, 36)];
+	       T2M = ri[WS(ios, 2)];
+	       T2g = ri[WS(ios, 60)];
+	       T2b = ii[WS(ios, 52)];
+	       T29 = ri[WS(ios, 52)];
+	       T26 = ii[WS(ios, 20)];
+	       T22 = ri[WS(ios, 20)];
+	       T2i = ii[WS(ios, 60)];
+	       T2E = ri[WS(ios, 44)];
+	       T2G = ii[WS(ios, 44)];
+	       T2v = ii[WS(ios, 12)];
+	       T2u = ri[WS(ios, 12)];
+	       T2r = ii[WS(ios, 28)];
+	       T2n = ri[WS(ios, 28)];
+	       T2N = ii[WS(ios, 2)];
+	       T3a = ii[WS(ios, 10)];
+	       T38 = ri[WS(ios, 10)];
+	       T3l = ri[WS(ios, 42)];
+	       T3n = ii[WS(ios, 42)];
+	       T3r = ri[WS(ios, 58)];
+	       T3t = ii[WS(ios, 58)];
+	       T33 = ii[WS(ios, 50)];
+	       T31 = ri[WS(ios, 50)];
+	       T2Y = ii[WS(ios, 18)];
+	       T2W = ri[WS(ios, 18)];
+	       T4g = ri[WS(ios, 54)];
+	       T2S = ii[WS(ios, 34)];
+	       T2Q = ri[WS(ios, 34)];
+	       T3w = ri[WS(ios, 26)];
+	       T3y = ii[WS(ios, 26)];
+	       T3E = ri[WS(ios, 62)];
+	       T3G = ii[WS(ios, 62)];
+	       T3P = ri[WS(ios, 14)];
+	       T3J = ri[WS(ios, 30)];
+	       T3L = ii[WS(ios, 30)];
+	       T3R = ii[WS(ios, 14)];
+	       T4a = ri[WS(ios, 38)];
+	       T4c = ii[WS(ios, 38)];
+	       T47 = ii[WS(ios, 6)];
+	       T46 = ri[WS(ios, 6)];
+	       T42 = ii[WS(ios, 46)];
+	       T40 = ri[WS(ios, 46)];
+	       T4i = ii[WS(ios, 54)];
+	       T6P = ri[WS(ios, 51)];
+	       T6R = ii[WS(ios, 51)];
+	       T6M = ii[WS(ios, 19)];
+	       T6L = ri[WS(ios, 19)];
+	       T6I = ii[WS(ios, 35)];
+	       T6G = ri[WS(ios, 35)];
+	       T6W = ri[WS(ios, 59)];
+	       T6Y = ii[WS(ios, 59)];
+	       T74 = ri[WS(ios, 11)];
+	       T75 = ii[WS(ios, 11)];
+	       T5I = ii[WS(ios, 13)];
+	       T78 = ri[WS(ios, 43)];
+	       T7a = ii[WS(ios, 43)];
+	       T6x = ii[WS(ios, 23)];
+	       T6v = ri[WS(ios, 23)];
+	       T6s = ii[WS(ios, 55)];
+	       T6q = ri[WS(ios, 55)];
+	       T6h = ii[WS(ios, 7)];
+	       T6m = ii[WS(ios, 39)];
+	       T6k = ri[WS(ios, 39)];
+	       T6g = ri[WS(ios, 7)];
+	       T5N = ii[WS(ios, 45)];
+	       T5L = ri[WS(ios, 45)];
+	       T5Z = ri[WS(ios, 31)];
+	       T63 = ii[WS(ios, 31)];
+	       T66 = ri[WS(ios, 15)];
+	       T67 = ii[WS(ios, 15)];
+	       T5H = ri[WS(ios, 13)];
+	       T54 = ri[WS(ios, 25)];
+	       T4D = ri[WS(ios, 17)];
+	       T4A = ii[WS(ios, 33)];
+	       T4y = ri[WS(ios, 33)];
+	       T4n = ii[WS(ios, 22)];
+	       T4l = ri[WS(ios, 22)];
+	       T4E = ii[WS(ios, 17)];
+	       T4X = ri[WS(ios, 57)];
+	       T4Z = ii[WS(ios, 57)];
+	       T4T = ii[WS(ios, 41)];
+	       T4R = ri[WS(ios, 41)];
+	       T4O = ii[WS(ios, 9)];
+	       T4N = ri[WS(ios, 9)];
+	       T58 = ii[WS(ios, 25)];
+	       T5s = ii[WS(ios, 53)];
+	       T5q = ri[WS(ios, 53)];
+	       T5x = ri[WS(ios, 61)];
+	       T5z = ii[WS(ios, 61)];
+	       T5C = ri[WS(ios, 29)];
+	       T5E = ii[WS(ios, 29)];
+	       T5n = ii[WS(ios, 21)];
+	       T5m = ri[WS(ios, 21)];
+	       T5j = ii[WS(ios, 37)];
+	       T5h = ri[WS(ios, 37)];
+	       T5d = ri[WS(ios, 5)];
+	       T5e = ii[WS(ios, 5)];
+	       {
+		    E T4u, T4v, T5T, T5U, T6C, T6D, T70, T71, T4, T7, T5, T8, TO, TP, T1U;
+		    E T2p, T18, T2k, T2l, T2o, TT, TS, T19, T1c, T1T, T1P, T1Q, T1d;
+		    T4u = ri[WS(ios, 1)];
+		    T4v = ii[WS(ios, 1)];
+		    T5T = ri[WS(ios, 63)];
+		    T5U = ii[WS(ios, 63)];
+		    T6C = ri[WS(ios, 3)];
+		    T6D = ii[WS(ios, 3)];
+		    T70 = ri[WS(ios, 27)];
+		    T71 = ii[WS(ios, 27)];
+		    {
+			 E T6, Te, T9, Td;
+			 T4 = W[2];
+			 T7 = W[3];
+			 T5 = W[0];
+			 T8 = W[1];
+			 T6 = T4 * T5;
+			 Te = T7 * T5;
+			 T9 = T7 * T8;
+			 Td = T4 * T8;
+			 Ta = T6 - T9;
+			 Ty = Td - Te;
+			 Tf = Td + Te;
+			 Tw = T6 + T9;
+			 T2 = W[6];
+			 Tj = W[7];
+			 T3 = W[4];
+			 Tc = W[5];
+			 TO = T3 * T4;
+			 TP = Tc * T7;
+			 T1U = Tj * T3;
+			 T2p = Tj * T5;
+			 T18 = T3 * T5;
+			 T2k = T2 * T5;
+			 T2l = Tj * T8;
+			 T2o = T2 * T8;
+			 TT = Tc * T4;
+			 TS = T3 * T7;
+			 T19 = Tc * T8;
+			 T1c = T3 * T8;
+			 T1E = T2 * T7;
+			 T1T = T2 * Tc;
+			 T1B = Tj * T7;
+			 T1F = Tj * T4;
+			 T1P = T2 * T3;
+			 T1Q = Tj * Tc;
+			 T1A = T2 * T4;
+			 T1d = Tc * T5;
+		    }
+		    T1R = T1P - T1Q;
+		    T3x = T2o - T2p;
+		    T2m = T2k - T2l;
+		    T3K = T1E + T1F;
+		    T61 = Tj * Ta;
+		    T1V = T1T + T1U;
+		    T60 = T2 * Tf;
+		    T3I = T1A - T1B;
+		    T51 = T2 * Tw;
+		    T52 = Tj * Ty;
+		    T2V = T1P + T1Q;
+		    T56 = Tj * Tw;
+		    T5X = Tj * Tf;
+		    T3v = T2k + T2l;
+		    T55 = T2 * Ty;
+		    T2X = T1T - T1U;
+		    T2q = T2o + T2p;
+		    T5W = T2 * Ta;
+		    T4w = FMA(T5, T4u, T8 * T4v);
+		    T6E = FMA(T4, T6C, T7 * T6D);
+		    Ta0 = FNMS(T7, T6C, T4 * T6D);
+		    T8Q = FNMS(T8, T4u, T5 * T4v);
+		    Tac = FNMS(Tj, T70, T2 * T71);
+		    T72 = FMA(T2, T70, Tj * T71);
+		    Tb = T3 * Ta;
+		    Tg = Tc * Tf;
+		    Th = Tb + Tg;
+		    T3e = TS - TT;
+		    T3f = Tj * T3e;
+		    T3h = T2 * T3e;
+		    T1a = T18 + T19;
+		    T2x = T2 * T1a;
+		    T2B = Tj * T1a;
+		    TU = TS + TT;
+		    TV = Tj * TU;
+		    TY = T2 * TU;
+		    T1e = T1c - T1d;
+		    T2y = Tj * T1e;
+		    T2A = T2 * T1e;
+		    TC = T3 * Ty;
+		    TD = Tc * Tw;
+		    T1u = TC + TD;
+		    Tk = T3 * Tf;
+		    Tl = Tc * Ta;
+		    Tm = Tk - Tl;
+		    T39 = T1c + T1d;
+		    T3U = Tj * T39;
+		    T3W = T2 * T39;
+		    T37 = T18 - T19;
+		    T3T = T2 * T37;
+		    T3X = Tj * T37;
+		    TQ = TO - TP;
+		    TR = T2 * TQ;
+		    TZ = Tj * TQ;
+		    T3c = TO + TP;
+		    T3d = T2 * T3c;
+		    T3i = Tj * T3c;
+		    Tx = T3 * Tw;
+		    Tz = Tc * Ty;
+		    T1t = Tx - Tz;
+		    TN = W[8];
+		    TX = W[9];
+		    T2f = FMA(TN, T4, TX * T7);
+		    T5V = FMA(TN, T5T, TX * T5U);
+		    Tao = FNMS(TX, T5T, TN * T5U);
+		    T2h = FNMS(TX, T4, TN * T7);
+		    T3D = FMA(TN, T5, TX * T8);
+		    T4f = FMA(TN, T3, TX * Tc);
+		    T4h = FNMS(TX, T3, TN * Tc);
+		    T3F = FNMS(TX, T5, TN * T8);
+	       }
+	       T1g = FNMS(T1e, T1f, T1a * T1b);
+	       T91 = FNMS(Tc, T4N, T3 * T4O);
+	       T7W = FMA(Ty, T2M, Tw * T2N);
+	       T7m = FMA(T1e, T1b, T1a * T1f);
+	       T2O = FNMS(Ty, T2N, Tw * T2M);
+	       T4j = FNMS(T4h, T4i, T4f * T4g);
+	       T7P = FNMS(TU, T2u, TQ * T2v);
+	       T4P = FMA(T3, T4N, Tc * T4O);
+	       T8y = FMA(T3e, T46, T3c * T47);
+	       T2w = FMA(TQ, T2u, TU * T2v);
+	       {
+		    E T1v, T1x, T1O, T1X;
+		    T8t = FMA(T4h, T4g, T4f * T4i);
+		    T2Z = FNMS(T2X, T2Y, T2V * T2W);
+		    T8e = FMA(T2X, T2W, T2V * T2Y);
+		    T48 = FNMS(T3e, T47, T3c * T46);
+		    T1v = FMA(TN, T1t, TX * T1u);
+		    T1x = FNMS(TX, T1t, TN * T1u);
+		    T1z = FNMS(T1x, T1y, T1v * T1w);
+		    T7s = FMA(T1x, T1w, T1v * T1y);
+		    {
+			 E T1C, T1G, T8n, T8o;
+			 T1C = T1A + T1B;
+			 T1G = T1E - T1F;
+			 T1I = FNMS(T1G, T1H, T1C * T1D);
+			 T7t = FMA(T1G, T1D, T1C * T1H);
+			 T8n = FMA(T3F, T3E, T3D * T3G);
+			 T8o = FNMS(T3K, T3J, T3I * T3L);
+			 T8p = T8n - T8o;
+			 Ten = T8n + T8o;
+		    }
+		    T1O = FMA(Ta, T1M, Tf * T1N);
+		    T1X = FMA(T1R, T1S, T1V * T1W);
+		    T1Y = T1O + T1X;
+		    T7D = T1O - T1X;
+		    {
+			 E T2j, T2s, T7J, T7K;
+			 T2j = FNMS(T2h, T2i, T2f * T2g);
+			 T2s = FMA(T2m, T2n, T2q * T2r);
+			 T2t = T2j + T2s;
+			 T7O = T2j - T2s;
+			 T7J = FMA(T2h, T2g, T2f * T2i);
+			 T7K = FNMS(T2q, T2n, T2m * T2r);
+			 T7L = T7J - T7K;
+			 Te6 = T7J + T7K;
+		    }
+	       }
+	       {
+		    E T3H, T3M, T7y, T7z;
+		    T3H = FNMS(T3F, T3G, T3D * T3E);
+		    T3M = FMA(T3I, T3J, T3K * T3L);
+		    T3N = T3H + T3M;
+		    T8E = T3H - T3M;
+		    T7y = FNMS(Tf, T1M, Ta * T1N);
+		    T7z = FNMS(T1V, T1S, T1R * T1W);
+		    T7A = T7y - T7z;
+		    Te0 = T7y + T7z;
+	       }
+	       {
+		    E T4B, T8R, T4x, T4z;
+		    T4x = T3d + T3f;
+		    T4z = T3h - T3i;
+		    T4B = FNMS(T4z, T4A, T4x * T4y);
+		    T8R = FMA(T4z, T4y, T4x * T4A);
+		    T4C = T4w + T4B;
+		    TeA = T8Q + T8R;
+		    T8S = T8Q - T8R;
+		    T9v = T4w - T4B;
+	       }
+	       {
+		    E T64, Tap, T5Y, T62;
+		    T5Y = T5W - T5X;
+		    T62 = T60 + T61;
+		    T64 = FMA(T5Y, T5Z, T62 * T63);
+		    Tap = FNMS(T62, T5Z, T5Y * T63);
+		    T65 = T5V + T64;
+		    Tfi = Tao + Tap;
+		    T9J = T5V - T64;
+		    Taq = Tao - Tap;
+	       }
+	       {
+		    E T6J, Ta1, T6F, T6H;
+		    T6F = T2x + T2y;
+		    T6H = T2A - T2B;
+		    T6J = FNMS(T6H, T6I, T6F * T6G);
+		    Ta1 = FMA(T6H, T6G, T6F * T6I);
+		    T6K = T6E + T6J;
+		    Tf6 = Ta0 + Ta1;
+		    Ta2 = Ta0 - Ta1;
+		    Ta5 = T6E - T6J;
+	       }
+	       {
+		    E T6Z, Tab, T6V, T6X;
+		    T6V = FMA(TN, Ta, TX * Tf);
+		    T6X = FNMS(TX, Ta, TN * Tf);
+		    T6Z = FNMS(T6X, T6Y, T6V * T6W);
+		    Tab = FMA(T6X, T6W, T6V * T6Y);
+		    T73 = T6Z + T72;
+		    Tfc = Tab + Tac;
+		    Tad = Tab - Tac;
+		    Tag = T6Z - T72;
+	       }
+	       T3z = FNMS(T3x, T3y, T3v * T3w);
+	       T83 = FMA(T3x, T3w, T3v * T3y);
+	       T3q = FNMS(TX, Tm, TN * Th);
+	       T3s = FMA(TN, Tm, TX * Th);
+	       T3u = FMA(T3q, T3r, T3s * T3t);
+	       T82 = FNMS(T3s, T3r, T3q * T3t);
+	       T81 = T3u - T3z;
+	       T84 = T82 - T83;
+	       {
+		    E TW, T10, T11, T13;
+		    TW = TR + TV;
+		    T10 = TY - TZ;
+		    T11 = FNMS(TX, T10, TN * TW);
+		    T13 = FMA(TN, T10, TX * TW);
+		    T15 = FMA(T11, T12, T13 * T14);
+		    T9K = FMA(T10, T66, TW * T67);
+		    T68 = FNMS(T10, T67, TW * T66);
+		    T7j = FNMS(T13, T12, T11 * T14);
+	       }
+	       {
+		    E T3V, T3Y, T3Z, T41;
+		    T3V = T3T + T3U;
+		    T3Y = T3W - T3X;
+		    T3Z = FNMS(TX, T3Y, TN * T3V);
+		    T41 = FMA(TN, T3Y, TX * T3V);
+		    T43 = FMA(T3Z, T40, T41 * T42);
+		    T9w = FMA(T3Y, T4D, T3V * T4E);
+		    T4F = FNMS(T3Y, T4E, T3V * T4D);
+		    T8G = FNMS(T41, T40, T3Z * T42);
+	       }
+	       {
+		    E T5f, T9i, T5k, T9j, T5g, T5i;
+		    T5f = FNMS(Tm, T5e, Th * T5d);
+		    T9i = FMA(Tm, T5d, Th * T5e);
+		    T5g = T3T - T3U;
+		    T5i = T3W + T3X;
+		    T5k = FMA(T5g, T5h, T5i * T5j);
+		    T9j = FNMS(T5i, T5h, T5g * T5j);
+		    T5l = T5f + T5k;
+		    TeL = T9i + T9j;
+		    T9k = T9i - T9j;
+		    T9n = T5f - T5k;
+	       }
+	       {
+		    E T6i, T9O, T6n, T9P, T6j, T6l;
+		    T6i = FMA(T1t, T6g, T1u * T6h);
+		    T9O = FNMS(T1u, T6g, T1t * T6h);
+		    T6j = TR - TV;
+		    T6l = TY + TZ;
+		    T6n = FMA(T6j, T6k, T6l * T6m);
+		    T9P = FNMS(T6l, T6k, T6j * T6m);
+		    T6o = T6i + T6n;
+		    Tf2 = T9O + T9P;
+		    T9Q = T9O - T9P;
+		    T9R = T6i - T6n;
+	       }
+	       {
+		    E T6t, T9U, T6y, T9V;
+		    {
+			 E T6p, T6r, T6u, T6w;
+			 T6p = FNMS(TX, T1e, TN * T1a);
+			 T6r = FMA(TN, T1e, TX * T1a);
+			 T6t = FMA(T6p, T6q, T6r * T6s);
+			 T9U = FNMS(T6r, T6q, T6p * T6s);
+			 T6u = T5W + T5X;
+			 T6w = T60 - T61;
+			 T6y = FNMS(T6w, T6x, T6u * T6v);
+			 T9V = FMA(T6w, T6v, T6u * T6x);
+		    }
+		    T6z = T6t + T6y;
+		    Tf3 = T9U + T9V;
+		    T9T = T6t - T6y;
+		    T9W = T9U - T9V;
+	       }
+	       {
+		    E Ti, Tn, T4k, Tq, Tr, T4m, T4Q, T4S;
+		    Ti = T2 * Th;
+		    Tn = Tj * Tm;
+		    T4k = Ti - Tn;
+		    Tq = T2 * Tm;
+		    Tr = Tj * Th;
+		    T4m = Tq + Tr;
+		    To = Ti + Tn;
+		    Ts = Tq - Tr;
+		    T4o = FMA(T4k, T4l, T4m * T4n);
+		    T8u = FNMS(T4m, T4l, T4k * T4n);
+		    T4Q = FMA(TN, T4k, TX * T4m);
+		    T4S = FNMS(TX, T4k, TN * T4m);
+		    T4U = FNMS(T4S, T4T, T4Q * T4R);
+		    T92 = FMA(T4S, T4R, T4Q * T4T);
+	       }
+	       {
+		    E T50, T8W, T59, T8X;
+		    {
+			 E T4W, T4Y, T53, T57;
+			 T4W = FNMS(TX, T3e, TN * T3c);
+			 T4Y = FMA(TN, T3e, TX * T3c);
+			 T50 = FMA(T4W, T4X, T4Y * T4Z);
+			 T8W = FNMS(T4Y, T4X, T4W * T4Z);
+			 T53 = T51 - T52;
+			 T57 = T55 + T56;
+			 T59 = FMA(T53, T54, T57 * T58);
+			 T8X = FNMS(T57, T54, T53 * T58);
+		    }
+		    T5a = T50 + T59;
+		    TeT = T8W + T8X;
+		    T8V = T50 - T59;
+		    T8Y = T8W - T8X;
+	       }
+	       {
+		    E T5A, T9c, T5F, T9d;
+		    {
+			 E T5w, T5y, T5B, T5D;
+			 T5w = FNMS(TX, Ty, TN * Tw);
+			 T5y = FMA(TN, Ty, TX * Tw);
+			 T5A = FMA(T5w, T5x, T5y * T5z);
+			 T9c = FNMS(T5y, T5x, T5w * T5z);
+			 T5B = T51 + T52;
+			 T5D = T55 - T56;
+			 T5F = FNMS(T5D, T5E, T5B * T5C);
+			 T9d = FMA(T5D, T5C, T5B * T5E);
+		    }
+		    T5G = T5A + T5F;
+		    TeG = T9c + T9d;
+		    T97 = T5A - T5F;
+		    T9e = T9c - T9d;
+	       }
+	       {
+		    E T21, T2P, T25, T2R, T77, T79;
+		    {
+			 E T1Z, T20, T23, T24;
+			 T1Z = T2 * T1t;
+			 T20 = Tj * T1u;
+			 T21 = T1Z + T20;
+			 T2P = T1Z - T20;
+			 T23 = T2 * T1u;
+			 T24 = Tj * T1t;
+			 T25 = T23 - T24;
+			 T2R = T23 + T24;
+		    }
+		    T27 = FNMS(T25, T26, T21 * T22);
+		    T7X = FNMS(T2R, T2Q, T2P * T2S);
+		    T2T = FMA(T2P, T2Q, T2R * T2S);
+		    T7E = FMA(T25, T22, T21 * T26);
+		    T77 = FNMS(TX, T25, TN * T21);
+		    T79 = FMA(TN, T25, TX * T21);
+		    T7b = FMA(T77, T78, T79 * T7a);
+		    Tai = FNMS(T79, T78, T77 * T7a);
+	       }
+	       {
+		    E T6S, Ta7, T2D, Ta6, T2F, T6N;
+		    {
+			 E T6O, T6Q, T2z, T2C;
+			 T6O = FMA(TN, TQ, TX * TU);
+			 T6Q = FNMS(TX, TQ, TN * TU);
+			 T6S = FNMS(T6Q, T6R, T6O * T6P);
+			 Ta7 = FMA(T6Q, T6P, T6O * T6R);
+			 T2z = T2x - T2y;
+			 T2C = T2A + T2B;
+			 T2D = FMA(TN, T2z, TX * T2C);
+			 Ta6 = FNMS(T2C, T6L, T2z * T6M);
+			 T2F = FNMS(TX, T2z, TN * T2C);
+			 T6N = FMA(T2z, T6L, T2C * T6M);
+		    }
+		    T6T = T6N + T6S;
+		    Ta3 = T6N - T6S;
+		    Tf7 = Ta6 + Ta7;
+		    Ta8 = Ta6 - Ta7;
+		    T7Q = FMA(T2F, T2E, T2D * T2G);
+		    T2H = FNMS(T2F, T2G, T2D * T2E);
+	       }
+	       {
+		    E TA, TE, TB, TF, TJ, TI, T2a, T28, T49, T4b;
+		    TA = Tx + Tz;
+		    TE = TC - TD;
+		    TB = T2 * TA;
+		    TF = Tj * TE;
+		    TJ = Tj * TA;
+		    TI = T2 * TE;
+		    T2a = FMA(TN, TE, TX * TA);
+		    T28 = FNMS(TX, TE, TN * TA);
+		    T2c = FMA(T28, T29, T2a * T2b);
+		    T76 = FNMS(TE, T75, TA * T74);
+		    Tah = FMA(TE, T74, TA * T75);
+		    T7F = FNMS(T2a, T29, T28 * T2b);
+		    T49 = TB + TF;
+		    T4b = TI - TJ;
+		    T4d = FNMS(T4b, T4c, T49 * T4a);
+		    T8z = FMA(T4b, T4a, T49 * T4c);
+		    TG = TB - TF;
+		    TK = TI + TJ;
+		    T69 = FMA(TN, TG, TX * TK);
+		    T6b = FNMS(TX, TG, TN * TK);
+	       }
+	       {
+		    E T5t, T9p, T3k, T9o, T3m, T5o;
+		    T3b = FMA(T37, T38, T39 * T3a);
+		    T87 = FNMS(T39, T38, T37 * T3a);
+		    {
+			 E T5p, T5r, T3g, T3j;
+			 T5p = FMA(TN, T37, TX * T39);
+			 T5r = FNMS(TX, T37, TN * T39);
+			 T5t = FNMS(T5r, T5s, T5p * T5q);
+			 T9p = FMA(T5r, T5q, T5p * T5s);
+			 T3g = T3d - T3f;
+			 T3j = T3h + T3i;
+			 T3k = FMA(TN, T3g, TX * T3j);
+			 T9o = FNMS(T3j, T5m, T3g * T5n);
+			 T3m = FNMS(TX, T3g, TN * T3j);
+			 T5o = FMA(T3g, T5m, T3j * T5n);
+		    }
+		    T5u = T5o + T5t;
+		    T9l = T5o - T5t;
+		    TeM = T9o + T9p;
+		    T9q = T9o - T9p;
+		    T88 = FMA(T3m, T3l, T3k * T3n);
+		    T89 = T87 - T88;
+		    T3o = FNMS(T3m, T3n, T3k * T3l);
+		    T86 = T3b - T3o;
+	       }
+	       {
+		    E T5O, T99, T1i, T1n, T1o, T1k, T30, T5J, T98, T32;
+		    {
+			 E T5K, T5M, T1h, T1j;
+			 T5K = FNMS(TX, T2X, TN * T2V);
+			 T5M = FMA(TN, T2X, TX * T2V);
+			 T5O = FMA(T5K, T5L, T5M * T5N);
+			 T99 = FNMS(T5M, T5L, T5K * T5N);
+			 T1h = Tb - Tg;
+			 T1j = Tk + Tl;
+			 T1i = T2 * T1h;
+			 T1n = T2 * T1j;
+			 T1o = Tj * T1h;
+			 T1k = Tj * T1j;
+			 T30 = FMA(TN, T1h, TX * T1j);
+			 T5J = FMA(T1h, T5H, T1j * T5I);
+			 T98 = FNMS(T1j, T5H, T1h * T5I);
+			 T32 = FNMS(TX, T1h, TN * T1j);
+		    }
+		    T5P = T5J + T5O;
+		    T9f = T5J - T5O;
+		    TeH = T98 + T99;
+		    T9a = T98 - T99;
+		    T34 = FNMS(T32, T33, T30 * T31);
+		    T8f = FMA(T32, T31, T30 * T33);
+		    {
+			 E T1l, T1p, T3O, T3Q;
+			 T1l = T1i - T1k;
+			 T1p = T1n + T1o;
+			 T1r = FMA(T1l, T1m, T1p * T1q);
+			 T7n = FNMS(T1p, T1m, T1l * T1q);
+			 T3O = T1i + T1k;
+			 T3Q = T1n - T1o;
+			 T3S = FNMS(T3Q, T3R, T3O * T3P);
+			 T8F = FMA(T3Q, T3P, T3O * T3R);
+			 T4G = FNMS(TX, T3Q, TN * T3O);
+			 T4I = FMA(TN, T3Q, TX * T3O);
+		    }
+	       }
+	  }
+	  {
+	       E T5R, TgT, TgY, ThE, T9t, Tbe, T9G, Tbb, Tcl, Tdq, Tcs, Tdn, TeP, Tg4, TeY;
+	       E Tg1, T7e, Th4, ThJ, Th9, Tfp, Tg8, Tfg, Tgb, T2K, TgC, Tih, ThX, TfQ, TiL;
+	       E Tea, Tiv, Tam, Tbl, TcL, Tdu, Taz, Tbi, TcE, Tdx, T7U, Tjv, Tdc, Tjh, Tb0;
+	       E TjL, TbU, TiZ, T8D, Tb5, Tc8, Tdi, T8M, Tb6, Tc5, Tdh, T4r, Thz, Tex, Tfz;
+	       E TfX, Tgl, TgN, Thj, T8m, TaI, Tdg, TdG, Tb4, Tbu, Tc2, TcU, T3C, Thy, Tem;
+	       E Tfy, TfU, Tgk, TgI, Thi, T6B, Th1, Tfm, Tga, Th8, ThI, T9Z, Tbh, Taw, Tbk;
+	       E TcI, Tdw, Tf5, Tg7, Tcx, Tdt, T5c, TgV, TeV, Tg0, TgS, ThD, TeE, Tg3, T96;
+	       E Tbd, Tce, Tdp, Tcp, Tdm, T9D, Tba, T1L, Tgz, Ti4, Tii, Tiy, TiM, TdZ, TfN;
+	       E T7x, TaX, Tj4, Tji, Tjy, TjM, TbN, Td9;
+	       {
+		    E T5v, T5Q, TgW, TgX;
+		    T5v = T5l + T5u;
+		    T5Q = T5G + T5P;
+		    T5R = T5v + T5Q;
+		    TgT = T5Q - T5v;
+		    TgW = TeL + TeM;
+		    TgX = TeG + TeH;
+		    TgY = TgW - TgX;
+		    ThE = TgW + TgX;
+	       }
+	       {
+		    E T9h, T9F, T9s, T9E;
+		    {
+			 E T9b, T9g, T9m, T9r;
+			 T9b = T97 - T9a;
+			 T9g = T9e + T9f;
+			 T9h = FNMS(KP923879532, T9g, KP382683432 * T9b);
+			 T9F = FMA(KP382683432, T9g, KP923879532 * T9b);
+			 T9m = T9k + T9l;
+			 T9r = T9n - T9q;
+			 T9s = FMA(KP923879532, T9m, KP382683432 * T9r);
+			 T9E = FNMS(KP923879532, T9r, KP382683432 * T9m);
+		    }
+		    T9t = T9h - T9s;
+		    Tbe = T9E + T9F;
+		    T9G = T9E - T9F;
+		    Tbb = T9s + T9h;
+	       }
+	       {
+		    E Tch, Tcr, Tck, Tcq;
+		    {
+			 E Tcf, Tcg, Tci, Tcj;
+			 Tcf = T97 + T9a;
+			 Tcg = T9e - T9f;
+			 Tch = FNMS(KP382683432, Tcg, KP923879532 * Tcf);
+			 Tcr = FMA(KP923879532, Tcg, KP382683432 * Tcf);
+			 Tci = T9k - T9l;
+			 Tcj = T9n + T9q;
+			 Tck = FMA(KP382683432, Tci, KP923879532 * Tcj);
+			 Tcq = FNMS(KP382683432, Tcj, KP923879532 * Tci);
+		    }
+		    Tcl = Tch - Tck;
+		    Tdq = Tcq + Tcr;
+		    Tcs = Tcq - Tcr;
+		    Tdn = Tck + Tch;
+	       }
+	       {
+		    E TeJ, TeX, TeO, TeW;
+		    {
+			 E TeF, TeI, TeK, TeN;
+			 TeF = T5G - T5P;
+			 TeI = TeG - TeH;
+			 TeJ = TeF - TeI;
+			 TeX = TeF + TeI;
+			 TeK = T5l - T5u;
+			 TeN = TeL - TeM;
+			 TeO = TeK + TeN;
+			 TeW = TeN - TeK;
+		    }
+		    TeP = KP707106781 * (TeJ - TeO);
+		    Tg4 = KP707106781 * (TeW + TeX);
+		    TeY = KP707106781 * (TeW - TeX);
+		    Tg1 = KP707106781 * (TeO + TeJ);
+	       }
+	       {
+		    E T6U, Th2, T7d, Tfb, Tfe, Th3, Tfa, Tfo, Tfn, Tff;
+		    T6U = T6K + T6T;
+		    Th2 = Tf6 + Tf7;
+		    {
+			 E T7c, Tfd, Tf8, Tf9;
+			 T7c = T76 + T7b;
+			 T7d = T73 + T7c;
+			 Tfb = T73 - T7c;
+			 Tfd = Tah + Tai;
+			 Tfe = Tfc - Tfd;
+			 Th3 = Tfc + Tfd;
+			 Tf8 = Tf6 - Tf7;
+			 Tf9 = T6K - T6T;
+			 Tfa = Tf8 - Tf9;
+			 Tfo = Tf9 + Tf8;
+		    }
+		    T7e = T6U + T7d;
+		    Th4 = Th2 - Th3;
+		    ThJ = Th2 + Th3;
+		    Th9 = T7d - T6U;
+		    Tfn = Tfb - Tfe;
+		    Tfp = KP707106781 * (Tfn - Tfo);
+		    Tg8 = KP707106781 * (Tfo + Tfn);
+		    Tff = Tfb + Tfe;
+		    Tfg = KP707106781 * (Tfa - Tff);
+		    Tgb = KP707106781 * (Tfa + Tff);
+	       }
+	       {
+		    E T2e, Te3, Te8, TgB, T2J, Te5, Te2, TgA;
+		    {
+			 E T2d, Te7, T2I, Te1;
+			 T2d = T27 + T2c;
+			 T2e = T1Y + T2d;
+			 Te3 = T1Y - T2d;
+			 Te7 = T7P + T7Q;
+			 Te8 = Te6 - Te7;
+			 TgB = Te6 + Te7;
+			 T2I = T2w + T2H;
+			 T2J = T2t + T2I;
+			 Te5 = T2t - T2I;
+			 Te1 = T7E + T7F;
+			 Te2 = Te0 - Te1;
+			 TgA = Te0 + Te1;
+		    }
+		    T2K = T2e + T2J;
+		    TgC = TgA - TgB;
+		    Tih = T2J - T2e;
+		    ThX = TgA + TgB;
+		    {
+			 E TfO, TfP, Te4, Te9;
+			 TfO = Te3 + Te2;
+			 TfP = Te5 - Te8;
+			 TfQ = KP707106781 * (TfO + TfP);
+			 TiL = KP707106781 * (TfP - TfO);
+			 Te4 = Te2 - Te3;
+			 Te9 = Te5 + Te8;
+			 Tea = KP707106781 * (Te4 - Te9);
+			 Tiv = KP707106781 * (Te4 + Te9);
+		    }
+	       }
+	       {
+		    E Taf, TcB, Tak, TcC, Taa, Tay, TcA, TcK, Tae, Taj;
+		    Tae = T76 - T7b;
+		    Taf = Tad + Tae;
+		    TcB = Tad - Tae;
+		    Taj = Tah - Tai;
+		    Tak = Tag - Taj;
+		    TcC = Tag + Taj;
+		    {
+			 E Ta4, Ta9, Tcy, Tcz;
+			 Ta4 = Ta2 + Ta3;
+			 Ta9 = Ta5 - Ta8;
+			 Taa = FNMS(KP923879532, Ta9, KP382683432 * Ta4);
+			 Tay = FMA(KP923879532, Ta4, KP382683432 * Ta9);
+			 Tcy = Ta2 - Ta3;
+			 Tcz = Ta5 + Ta8;
+			 TcA = FNMS(KP382683432, Tcz, KP923879532 * Tcy);
+			 TcK = FMA(KP382683432, Tcy, KP923879532 * Tcz);
+		    }
+		    {
+			 E Tal, TcJ, Tax, TcD;
+			 Tal = FMA(KP382683432, Taf, KP923879532 * Tak);
+			 Tam = Taa - Tal;
+			 Tbl = Taa + Tal;
+			 TcJ = FNMS(KP382683432, TcB, KP923879532 * TcC);
+			 TcL = TcJ - TcK;
+			 Tdu = TcK + TcJ;
+			 Tax = FNMS(KP923879532, Taf, KP382683432 * Tak);
+			 Taz = Tax - Tay;
+			 Tbi = Tay + Tax;
+			 TcD = FMA(KP923879532, TcB, KP382683432 * TcC);
+			 TcE = TcA - TcD;
+			 Tdx = TcA + TcD;
+		    }
+	       }
+	       {
+		    E T7C, TbO, T7S, TbS, T7H, TbP, T7N, TbR;
+		    {
+			 E T7B, T7R, T7G, T7M;
+			 T7B = T27 - T2c;
+			 T7C = T7A + T7B;
+			 TbO = T7A - T7B;
+			 T7R = T7P - T7Q;
+			 T7S = T7O - T7R;
+			 TbS = T7O + T7R;
+			 T7G = T7E - T7F;
+			 T7H = T7D - T7G;
+			 TbP = T7D + T7G;
+			 T7M = T2w - T2H;
+			 T7N = T7L + T7M;
+			 TbR = T7L - T7M;
+		    }
+		    {
+			 E T7I, T7T, Tda, Tdb;
+			 T7I = FNMS(KP923879532, T7H, KP382683432 * T7C);
+			 T7T = FMA(KP382683432, T7N, KP923879532 * T7S);
+			 T7U = T7I - T7T;
+			 Tjv = T7I + T7T;
+			 Tda = FMA(KP382683432, TbO, KP923879532 * TbP);
+			 Tdb = FNMS(KP382683432, TbR, KP923879532 * TbS);
+			 Tdc = Tda + Tdb;
+			 Tjh = Tdb - Tda;
+		    }
+		    {
+			 E TaY, TaZ, TbQ, TbT;
+			 TaY = FMA(KP923879532, T7C, KP382683432 * T7H);
+			 TaZ = FNMS(KP923879532, T7N, KP382683432 * T7S);
+			 Tb0 = TaY + TaZ;
+			 TjL = TaZ - TaY;
+			 TbQ = FNMS(KP382683432, TbP, KP923879532 * TbO);
+			 TbT = FMA(KP923879532, TbR, KP382683432 * TbS);
+			 TbU = TbQ - TbT;
+			 TiZ = TbQ + TbT;
+		    }
+	       }
+	       {
+		    E T8r, Tc6, T8I, Tc3, T8w, T8K, T8B, T8J, T8q, T8H;
+		    T8q = T3S - T43;
+		    T8r = T8p + T8q;
+		    Tc6 = T8p - T8q;
+		    T8H = T8F - T8G;
+		    T8I = T8E - T8H;
+		    Tc3 = T8E + T8H;
+		    {
+			 E T8s, T8v, T8x, T8A;
+			 T8s = T4j - T4o;
+			 T8v = T8t - T8u;
+			 T8w = T8s - T8v;
+			 T8K = T8s + T8v;
+			 T8x = T48 - T4d;
+			 T8A = T8y - T8z;
+			 T8B = T8x + T8A;
+			 T8J = T8A - T8x;
+		    }
+		    {
+			 E T8C, Tc7, T8L, Tc4;
+			 T8C = KP707106781 * (T8w - T8B);
+			 T8D = T8r - T8C;
+			 Tb5 = T8r + T8C;
+			 Tc7 = KP707106781 * (T8J + T8K);
+			 Tc8 = Tc6 - Tc7;
+			 Tdi = Tc6 + Tc7;
+			 T8L = KP707106781 * (T8J - T8K);
+			 T8M = T8I - T8L;
+			 Tb6 = T8I + T8L;
+			 Tc4 = KP707106781 * (T8B + T8w);
+			 Tc5 = Tc3 - Tc4;
+			 Tdh = Tc3 + Tc4;
+		    }
+	       }
+	       {
+		    E T45, Tes, Tep, TgK, T4q, Teq, Tev, TgL, T44, Teo, Ter, Tew;
+		    T44 = T3S + T43;
+		    T45 = T3N + T44;
+		    Tes = T3N - T44;
+		    Teo = T8F + T8G;
+		    Tep = Ten - Teo;
+		    TgK = Ten + Teo;
+		    {
+			 E T4e, T4p, Tet, Teu;
+			 T4e = T48 + T4d;
+			 T4p = T4j + T4o;
+			 T4q = T4e + T4p;
+			 Teq = T4p - T4e;
+			 Tet = T8y + T8z;
+			 Teu = T8t + T8u;
+			 Tev = Tet - Teu;
+			 TgL = Tet + Teu;
+		    }
+		    T4r = T45 + T4q;
+		    Thz = TgK + TgL;
+		    Ter = Tep - Teq;
+		    Tew = Tes - Tev;
+		    Tex = FMA(KP382683432, Ter, KP923879532 * Tew);
+		    Tfz = FNMS(KP923879532, Ter, KP382683432 * Tew);
+		    {
+			 E TfV, TfW, TgJ, TgM;
+			 TfV = Tep + Teq;
+			 TfW = Tes + Tev;
+			 TfX = FMA(KP923879532, TfV, KP382683432 * TfW);
+			 Tgl = FNMS(KP382683432, TfV, KP923879532 * TfW);
+			 TgJ = T45 - T4q;
+			 TgM = TgK - TgL;
+			 TgN = TgJ + TgM;
+			 Thj = TgJ - TgM;
+		    }
+	       }
+	       {
+		    E T80, TbW, T8k, TbX, T8b, Tc0, T8h, TbZ;
+		    {
+			 E T7Y, T7Z, T8i, T8j;
+			 T7Y = T7W - T7X;
+			 T7Z = T2Z - T34;
+			 T80 = T7Y + T7Z;
+			 TbW = T7Y - T7Z;
+			 T8i = T89 - T86;
+			 T8j = T81 + T84;
+			 T8k = KP707106781 * (T8i - T8j);
+			 TbX = KP707106781 * (T8i + T8j);
+		    }
+		    {
+			 E T85, T8a, T8d, T8g;
+			 T85 = T81 - T84;
+			 T8a = T86 + T89;
+			 T8b = KP707106781 * (T85 - T8a);
+			 Tc0 = KP707106781 * (T8a + T85);
+			 T8d = T2O - T2T;
+			 T8g = T8e - T8f;
+			 T8h = T8d - T8g;
+			 TbZ = T8d + T8g;
+		    }
+		    {
+			 E T8c, T8l, Tde, Tdf;
+			 T8c = T80 - T8b;
+			 T8l = T8h - T8k;
+			 T8m = FNMS(KP980785280, T8l, KP195090322 * T8c);
+			 TaI = FMA(KP980785280, T8c, KP195090322 * T8l);
+			 Tde = TbW + TbX;
+			 Tdf = TbZ + Tc0;
+			 Tdg = FNMS(KP195090322, Tdf, KP980785280 * Tde);
+			 TdG = FMA(KP980785280, Tdf, KP195090322 * Tde);
+		    }
+		    {
+			 E Tb2, Tb3, TbY, Tc1;
+			 Tb2 = T80 + T8b;
+			 Tb3 = T8h + T8k;
+			 Tb4 = FNMS(KP555570233, Tb3, KP831469612 * Tb2);
+			 Tbu = FMA(KP555570233, Tb2, KP831469612 * Tb3);
+			 TbY = TbW - TbX;
+			 Tc1 = TbZ - Tc0;
+			 Tc2 = FNMS(KP831469612, Tc1, KP555570233 * TbY);
+			 TcU = FMA(KP555570233, Tc1, KP831469612 * TbY);
+		    }
+	       }
+	       {
+		    E T36, Teh, Tek, TgF, T3B, Tef, Tee, TgE, Teg, Tel;
+		    {
+			 E T2U, T35, Tei, Tej;
+			 T2U = T2O + T2T;
+			 T35 = T2Z + T34;
+			 T36 = T2U + T35;
+			 Teh = T2U - T35;
+			 Tei = T87 + T88;
+			 Tej = T82 + T83;
+			 Tek = Tei - Tej;
+			 TgF = Tei + Tej;
+		    }
+		    {
+			 E T3p, T3A, Tec, Ted;
+			 T3p = T3b + T3o;
+			 T3A = T3u + T3z;
+			 T3B = T3p + T3A;
+			 Tef = T3A - T3p;
+			 Tec = T7W + T7X;
+			 Ted = T8e + T8f;
+			 Tee = Tec - Ted;
+			 TgE = Tec + Ted;
+		    }
+		    T3C = T36 + T3B;
+		    Thy = TgE + TgF;
+		    Teg = Tee - Tef;
+		    Tel = Teh - Tek;
+		    Tem = FNMS(KP923879532, Tel, KP382683432 * Teg);
+		    Tfy = FMA(KP923879532, Teg, KP382683432 * Tel);
+		    {
+			 E TfS, TfT, TgG, TgH;
+			 TfS = Tee + Tef;
+			 TfT = Teh + Tek;
+			 TfU = FNMS(KP382683432, TfT, KP923879532 * TfS);
+			 Tgk = FMA(KP382683432, TfS, KP923879532 * TfT);
+			 TgG = TgE - TgF;
+			 TgH = T36 - T3B;
+			 TgI = TgG - TgH;
+			 Thi = TgH + TgG;
+		    }
+	       }
+	       {
+		    E T6A, Tfl, Th7, Tf4, T6e, Tar, T9Y, TcH, Tav, Tcw, T9M, Tfj;
+		    T6A = T6o + T6z;
+		    Tfl = T6z - T6o;
+		    Th7 = Tf2 + Tf3;
+		    Tf4 = Tf2 - Tf3;
+		    {
+			 E T6d, T9S, T9X, Tat, Tau, T9L;
+			 T6d = FNMS(T6b, T6c, T69 * T6a);
+			 T6e = T68 + T6d;
+			 Tar = T68 - T6d;
+			 T9S = T9Q - T9R;
+			 T9X = T9T + T9W;
+			 T9Y = KP707106781 * (T9S - T9X);
+			 TcH = KP707106781 * (T9S + T9X);
+			 Tat = T9T - T9W;
+			 Tau = T9R + T9Q;
+			 Tav = KP707106781 * (Tat - Tau);
+			 Tcw = KP707106781 * (Tau + Tat);
+			 T9L = FMA(T6b, T6a, T69 * T6c);
+			 T9M = T9K - T9L;
+			 Tfj = T9K + T9L;
+		    }
+		    {
+			 E T6f, Tfk, Th6, T9N;
+			 T6f = T65 + T6e;
+			 T6B = T6f + T6A;
+			 Th1 = T6f - T6A;
+			 Tfk = Tfi - Tfj;
+			 Tfm = Tfk - Tfl;
+			 Tga = Tfk + Tfl;
+			 Th6 = Tfi + Tfj;
+			 Th8 = Th6 - Th7;
+			 ThI = Th6 + Th7;
+			 T9N = T9J - T9M;
+			 T9Z = T9N - T9Y;
+			 Tbh = T9N + T9Y;
+		    }
+		    {
+			 E Tas, TcG, Tf1, Tcv;
+			 Tas = Taq + Tar;
+			 Taw = Tas - Tav;
+			 Tbk = Tas + Tav;
+			 TcG = Taq - Tar;
+			 TcI = TcG - TcH;
+			 Tdw = TcG + TcH;
+			 Tf1 = T65 - T6e;
+			 Tf5 = Tf1 - Tf4;
+			 Tg7 = Tf1 + Tf4;
+			 Tcv = T9J + T9M;
+			 Tcx = Tcv - Tcw;
+			 Tdt = Tcv + Tcw;
+		    }
+	       }
+	       {
+		    E T8Z, T9B, T5b, TeD, TeU, TgR, T94, T9A, T4L, T8T, T9y, TeB, T4V;
+		    T8Z = T8V - T8Y;
+		    T9B = T8V + T8Y;
+		    T4V = T4P + T4U;
+		    T5b = T4V + T5a;
+		    TeD = T5a - T4V;
+		    {
+			 E TeS, T90, T93, T4K, T9x;
+			 TeS = T91 + T92;
+			 TeU = TeS - TeT;
+			 TgR = TeS + TeT;
+			 T90 = T4P - T4U;
+			 T93 = T91 - T92;
+			 T94 = T90 + T93;
+			 T9A = T93 - T90;
+			 T4K = FMA(T4G, T4H, T4I * T4J);
+			 T4L = T4F + T4K;
+			 T8T = T4F - T4K;
+			 T9x = FNMS(T4I, T4H, T4G * T4J);
+			 T9y = T9w - T9x;
+			 TeB = T9w + T9x;
+		    }
+		    {
+			 E T4M, TeR, TgQ, TeC;
+			 T4M = T4C + T4L;
+			 T5c = T4M + T5b;
+			 TgV = T4M - T5b;
+			 TeR = T4C - T4L;
+			 TeV = TeR - TeU;
+			 Tg0 = TeR + TeU;
+			 TgQ = TeA + TeB;
+			 TgS = TgQ - TgR;
+			 ThD = TgQ + TgR;
+			 TeC = TeA - TeB;
+			 TeE = TeC - TeD;
+			 Tg3 = TeC + TeD;
+		    }
+		    {
+			 E T8U, T95, Tcc, Tcd;
+			 T8U = T8S + T8T;
+			 T95 = KP707106781 * (T8Z - T94);
+			 T96 = T8U - T95;
+			 Tbd = T8U + T95;
+			 Tcc = T8S - T8T;
+			 Tcd = KP707106781 * (T9A + T9B);
+			 Tce = Tcc - Tcd;
+			 Tdp = Tcc + Tcd;
+		    }
+		    {
+			 E Tcn, Tco, T9z, T9C;
+			 Tcn = T9v + T9y;
+			 Tco = KP707106781 * (T94 + T8Z);
+			 Tcp = Tcn - Tco;
+			 Tdm = Tcn + Tco;
+			 T9z = T9v - T9y;
+			 T9C = KP707106781 * (T9A - T9B);
+			 T9D = T9z - T9C;
+			 Tba = T9z + T9C;
+		    }
+	       }
+	       {
+		    E Tv, T7h, TdY, ThY, Ti2, Tj1, T16, Tj2, T1K, Tiw, T7q, TbK, T7v, TbL, T7k;
+		    E ThZ, T7r, T7u, T7i;
+		    {
+			 E Tu, TdW, TdX, Ti0, TM;
+			 Tu = FNMS(Ts, Tt, To * Tp);
+			 Tv = T1 + Tu;
+			 T7h = T1 - Tu;
+			 TdW = T7m + T7n;
+			 TdX = T7s + T7t;
+			 TdY = TdW - TdX;
+			 ThY = TdW + TdX;
+			 Ti0 = FMA(Ts, Tp, To * Tt);
+			 Ti2 = Ti0 + Ti1;
+			 Tj1 = Ti1 - Ti0;
+			 TM = FMA(TG, TH, TK * TL);
+			 T16 = TM + T15;
+			 Tj2 = TM - T15;
+		    }
+		    {
+			 E T1s, T1J, T7o, T7p;
+			 T1s = T1g + T1r;
+			 T1J = T1z + T1I;
+			 T1K = T1s + T1J;
+			 Tiw = T1J - T1s;
+			 T7o = T7m - T7n;
+			 T7p = T1g - T1r;
+			 T7q = T7o - T7p;
+			 TbK = T7p + T7o;
+		    }
+		    T7r = T1z - T1I;
+		    T7u = T7s - T7t;
+		    T7v = T7r + T7u;
+		    TbL = T7r - T7u;
+		    T7i = FNMS(TK, TH, TG * TL);
+		    T7k = T7i - T7j;
+		    ThZ = T7i + T7j;
+		    {
+			 E T17, Ti3, Tix, TdV;
+			 T17 = Tv + T16;
+			 T1L = T17 + T1K;
+			 Tgz = T17 - T1K;
+			 Ti3 = ThZ + Ti2;
+			 Ti4 = ThY + Ti3;
+			 Tii = Ti3 - ThY;
+			 Tix = Ti2 - ThZ;
+			 Tiy = Tiw + Tix;
+			 TiM = Tix - Tiw;
+			 TdV = Tv - T16;
+			 TdZ = TdV - TdY;
+			 TfN = TdV + TdY;
+		    }
+		    {
+			 E T7l, T7w, Tj0, Tj3;
+			 T7l = T7h - T7k;
+			 T7w = KP707106781 * (T7q - T7v);
+			 T7x = T7l - T7w;
+			 TaX = T7l + T7w;
+			 Tj0 = KP707106781 * (T7q + T7v);
+			 Tj3 = Tj1 - Tj2;
+			 Tj4 = Tj0 + Tj3;
+			 Tji = Tj3 - Tj0;
+		    }
+		    {
+			 E Tjw, Tjx, TbJ, TbM;
+			 Tjw = KP707106781 * (TbL - TbK);
+			 Tjx = Tj2 + Tj1;
+			 Tjy = Tjw + Tjx;
+			 TjM = Tjx - Tjw;
+			 TbJ = T7h + T7k;
+			 TbM = KP707106781 * (TbK + TbL);
+			 TbN = TbJ - TbM;
+			 Td9 = TbJ + TbM;
+		    }
+	       }
+	       {
+		    E T4t, ThR, Ti6, Ti8, T7g, Ti7, ThU, ThV;
+		    {
+			 E T2L, T4s, ThW, Ti5;
+			 T2L = T1L + T2K;
+			 T4s = T3C + T4r;
+			 T4t = T2L + T4s;
+			 ThR = T2L - T4s;
+			 ThW = Thy + Thz;
+			 Ti5 = ThX + Ti4;
+			 Ti6 = ThW + Ti5;
+			 Ti8 = Ti5 - ThW;
+		    }
+		    {
+			 E T5S, T7f, ThS, ThT;
+			 T5S = T5c + T5R;
+			 T7f = T6B + T7e;
+			 T7g = T5S + T7f;
+			 Ti7 = T7f - T5S;
+			 ThS = ThD + ThE;
+			 ThT = ThI + ThJ;
+			 ThU = ThS - ThT;
+			 ThV = ThS + ThT;
+		    }
+		    ri[WS(ios, 32)] = T4t - T7g;
+		    ii[WS(ios, 32)] = Ti6 - ThV;
+		    ri[0] = T4t + T7g;
+		    ii[0] = ThV + Ti6;
+		    ri[WS(ios, 48)] = ThR - ThU;
+		    ii[WS(ios, 48)] = Ti8 - Ti7;
+		    ri[WS(ios, 16)] = ThR + ThU;
+		    ii[WS(ios, 16)] = Ti7 + Ti8;
+	       }
+	       {
+		    E ThB, ThN, Tic, Tie, ThG, ThO, ThL, ThP;
+		    {
+			 E Thx, ThA, Tia, Tib;
+			 Thx = T1L - T2K;
+			 ThA = Thy - Thz;
+			 ThB = Thx + ThA;
+			 ThN = Thx - ThA;
+			 Tia = T4r - T3C;
+			 Tib = Ti4 - ThX;
+			 Tic = Tia + Tib;
+			 Tie = Tib - Tia;
+		    }
+		    {
+			 E ThC, ThF, ThH, ThK;
+			 ThC = T5c - T5R;
+			 ThF = ThD - ThE;
+			 ThG = ThC + ThF;
+			 ThO = ThF - ThC;
+			 ThH = T6B - T7e;
+			 ThK = ThI - ThJ;
+			 ThL = ThH - ThK;
+			 ThP = ThH + ThK;
+		    }
+		    {
+			 E ThM, Ti9, ThQ, Tid;
+			 ThM = KP707106781 * (ThG + ThL);
+			 ri[WS(ios, 40)] = ThB - ThM;
+			 ri[WS(ios, 8)] = ThB + ThM;
+			 Ti9 = KP707106781 * (ThO + ThP);
+			 ii[WS(ios, 8)] = Ti9 + Tic;
+			 ii[WS(ios, 40)] = Tic - Ti9;
+			 ThQ = KP707106781 * (ThO - ThP);
+			 ri[WS(ios, 56)] = ThN - ThQ;
+			 ri[WS(ios, 24)] = ThN + ThQ;
+			 Tid = KP707106781 * (ThL - ThG);
+			 ii[WS(ios, 24)] = Tid + Tie;
+			 ii[WS(ios, 56)] = Tie - Tid;
+		    }
+	       }
+	       {
+		    E TgP, Thd, Tiq, Tis, Th0, The, Thb, Thf;
+		    {
+			 E TgD, TgO, Tio, Tip;
+			 TgD = Tgz - TgC;
+			 TgO = KP707106781 * (TgI - TgN);
+			 TgP = TgD + TgO;
+			 Thd = TgD - TgO;
+			 Tio = KP707106781 * (Thj - Thi);
+			 Tip = Tii - Tih;
+			 Tiq = Tio + Tip;
+			 Tis = Tip - Tio;
+		    }
+		    {
+			 E TgU, TgZ, Th5, Tha;
+			 TgU = TgS - TgT;
+			 TgZ = TgV - TgY;
+			 Th0 = FMA(KP923879532, TgU, KP382683432 * TgZ);
+			 The = FNMS(KP923879532, TgZ, KP382683432 * TgU);
+			 Th5 = Th1 - Th4;
+			 Tha = Th8 - Th9;
+			 Thb = FNMS(KP923879532, Tha, KP382683432 * Th5);
+			 Thf = FMA(KP382683432, Tha, KP923879532 * Th5);
+		    }
+		    {
+			 E Thc, Tin, Thg, Tir;
+			 Thc = Th0 + Thb;
+			 ri[WS(ios, 44)] = TgP - Thc;
+			 ri[WS(ios, 12)] = TgP + Thc;
+			 Tin = The + Thf;
+			 ii[WS(ios, 12)] = Tin + Tiq;
+			 ii[WS(ios, 44)] = Tiq - Tin;
+			 Thg = The - Thf;
+			 ri[WS(ios, 60)] = Thd - Thg;
+			 ri[WS(ios, 28)] = Thd + Thg;
+			 Tir = Thb - Th0;
+			 ii[WS(ios, 28)] = Tir + Tis;
+			 ii[WS(ios, 60)] = Tis - Tir;
+		    }
+	       }
+	       {
+		    E TfB, TfJ, TiO, TiQ, TfE, TfK, TfH, TfL;
+		    {
+			 E Tfx, TfA, TiK, TiN;
+			 Tfx = TdZ + Tea;
+			 TfA = Tfy + Tfz;
+			 TfB = Tfx + TfA;
+			 TfJ = Tfx - TfA;
+			 TiK = Tem + Tex;
+			 TiN = TiL + TiM;
+			 TiO = TiK + TiN;
+			 TiQ = TiN - TiK;
+		    }
+		    {
+			 E TfC, TfD, TfF, TfG;
+			 TfC = TeE + TeP;
+			 TfD = TeV + TeY;
+			 TfE = FMA(KP555570233, TfC, KP831469612 * TfD);
+			 TfK = FNMS(KP555570233, TfD, KP831469612 * TfC);
+			 TfF = Tf5 + Tfg;
+			 TfG = Tfm + Tfp;
+			 TfH = FNMS(KP555570233, TfG, KP831469612 * TfF);
+			 TfL = FMA(KP831469612, TfG, KP555570233 * TfF);
+		    }
+		    {
+			 E TfI, TiJ, TfM, TiP;
+			 TfI = TfE + TfH;
+			 ri[WS(ios, 38)] = TfB - TfI;
+			 ri[WS(ios, 6)] = TfB + TfI;
+			 TiJ = TfK + TfL;
+			 ii[WS(ios, 6)] = TiJ + TiO;
+			 ii[WS(ios, 38)] = TiO - TiJ;
+			 TfM = TfK - TfL;
+			 ri[WS(ios, 54)] = TfJ - TfM;
+			 ri[WS(ios, 22)] = TfJ + TfM;
+			 TiP = TfH - TfE;
+			 ii[WS(ios, 22)] = TiP + TiQ;
+			 ii[WS(ios, 54)] = TiQ - TiP;
+		    }
+	       }
+	       {
+		    E Thl, Tht, Tik, Tim, Tho, Thu, Thr, Thv;
+		    {
+			 E Thh, Thk, Tig, Tij;
+			 Thh = Tgz + TgC;
+			 Thk = KP707106781 * (Thi + Thj);
+			 Thl = Thh + Thk;
+			 Tht = Thh - Thk;
+			 Tig = KP707106781 * (TgI + TgN);
+			 Tij = Tih + Tii;
+			 Tik = Tig + Tij;
+			 Tim = Tij - Tig;
+		    }
+		    {
+			 E Thm, Thn, Thp, Thq;
+			 Thm = TgS + TgT;
+			 Thn = TgV + TgY;
+			 Tho = FMA(KP382683432, Thm, KP923879532 * Thn);
+			 Thu = FNMS(KP382683432, Thn, KP923879532 * Thm);
+			 Thp = Th1 + Th4;
+			 Thq = Th8 + Th9;
+			 Thr = FNMS(KP382683432, Thq, KP923879532 * Thp);
+			 Thv = FMA(KP923879532, Thq, KP382683432 * Thp);
+		    }
+		    {
+			 E Ths, Tif, Thw, Til;
+			 Ths = Tho + Thr;
+			 ri[WS(ios, 36)] = Thl - Ths;
+			 ri[WS(ios, 4)] = Thl + Ths;
+			 Tif = Thu + Thv;
+			 ii[WS(ios, 4)] = Tif + Tik;
+			 ii[WS(ios, 36)] = Tik - Tif;
+			 Thw = Thu - Thv;
+			 ri[WS(ios, 52)] = Tht - Thw;
+			 ri[WS(ios, 20)] = Tht + Thw;
+			 Til = Thr - Tho;
+			 ii[WS(ios, 20)] = Til + Tim;
+			 ii[WS(ios, 52)] = Tim - Til;
+		    }
+	       }
+	       {
+		    E Tez, Tft, TiU, TiW, Tf0, Tfu, Tfr, Tfv;
+		    {
+			 E Teb, Tey, TiS, TiT;
+			 Teb = TdZ - Tea;
+			 Tey = Tem - Tex;
+			 Tez = Teb + Tey;
+			 Tft = Teb - Tey;
+			 TiS = Tfz - Tfy;
+			 TiT = TiM - TiL;
+			 TiU = TiS + TiT;
+			 TiW = TiT - TiS;
+		    }
+		    {
+			 E TeQ, TeZ, Tfh, Tfq;
+			 TeQ = TeE - TeP;
+			 TeZ = TeV - TeY;
+			 Tf0 = FMA(KP980785280, TeQ, KP195090322 * TeZ);
+			 Tfu = FNMS(KP980785280, TeZ, KP195090322 * TeQ);
+			 Tfh = Tf5 - Tfg;
+			 Tfq = Tfm - Tfp;
+			 Tfr = FNMS(KP980785280, Tfq, KP195090322 * Tfh);
+			 Tfv = FMA(KP195090322, Tfq, KP980785280 * Tfh);
+		    }
+		    {
+			 E Tfs, TiR, Tfw, TiV;
+			 Tfs = Tf0 + Tfr;
+			 ri[WS(ios, 46)] = Tez - Tfs;
+			 ri[WS(ios, 14)] = Tez + Tfs;
+			 TiR = Tfu + Tfv;
+			 ii[WS(ios, 14)] = TiR + TiU;
+			 ii[WS(ios, 46)] = TiU - TiR;
+			 Tfw = Tfu - Tfv;
+			 ri[WS(ios, 62)] = Tft - Tfw;
+			 ri[WS(ios, 30)] = Tft + Tfw;
+			 TiV = Tfr - Tf0;
+			 ii[WS(ios, 30)] = TiV + TiW;
+			 ii[WS(ios, 62)] = TiW - TiV;
+		    }
+	       }
+	       {
+		    E TfZ, Tgf, TiG, TiI, Tg6, Tgg, Tgd, Tgh;
+		    {
+			 E TfR, TfY, TiE, TiF;
+			 TfR = TfN - TfQ;
+			 TfY = TfU - TfX;
+			 TfZ = TfR + TfY;
+			 Tgf = TfR - TfY;
+			 TiE = Tgl - Tgk;
+			 TiF = Tiy - Tiv;
+			 TiG = TiE + TiF;
+			 TiI = TiF - TiE;
+		    }
+		    {
+			 E Tg2, Tg5, Tg9, Tgc;
+			 Tg2 = Tg0 - Tg1;
+			 Tg5 = Tg3 - Tg4;
+			 Tg6 = FMA(KP555570233, Tg2, KP831469612 * Tg5);
+			 Tgg = FNMS(KP831469612, Tg2, KP555570233 * Tg5);
+			 Tg9 = Tg7 - Tg8;
+			 Tgc = Tga - Tgb;
+			 Tgd = FNMS(KP831469612, Tgc, KP555570233 * Tg9);
+			 Tgh = FMA(KP831469612, Tg9, KP555570233 * Tgc);
+		    }
+		    {
+			 E Tge, TiD, Tgi, TiH;
+			 Tge = Tg6 + Tgd;
+			 ri[WS(ios, 42)] = TfZ - Tge;
+			 ri[WS(ios, 10)] = TfZ + Tge;
+			 TiD = Tgg + Tgh;
+			 ii[WS(ios, 10)] = TiD + TiG;
+			 ii[WS(ios, 42)] = TiG - TiD;
+			 Tgi = Tgg - Tgh;
+			 ri[WS(ios, 58)] = Tgf - Tgi;
+			 ri[WS(ios, 26)] = Tgf + Tgi;
+			 TiH = Tgd - Tg6;
+			 ii[WS(ios, 26)] = TiH + TiI;
+			 ii[WS(ios, 58)] = TiI - TiH;
+		    }
+	       }
+	       {
+		    E Tgn, Tgv, TiA, TiC, Tgq, Tgw, Tgt, Tgx;
+		    {
+			 E Tgj, Tgm, Tiu, Tiz;
+			 Tgj = TfN + TfQ;
+			 Tgm = Tgk + Tgl;
+			 Tgn = Tgj + Tgm;
+			 Tgv = Tgj - Tgm;
+			 Tiu = TfU + TfX;
+			 Tiz = Tiv + Tiy;
+			 TiA = Tiu + Tiz;
+			 TiC = Tiz - Tiu;
+		    }
+		    {
+			 E Tgo, Tgp, Tgr, Tgs;
+			 Tgo = Tg0 + Tg1;
+			 Tgp = Tg3 + Tg4;
+			 Tgq = FMA(KP980785280, Tgo, KP195090322 * Tgp);
+			 Tgw = FNMS(KP195090322, Tgo, KP980785280 * Tgp);
+			 Tgr = Tg7 + Tg8;
+			 Tgs = Tga + Tgb;
+			 Tgt = FNMS(KP195090322, Tgs, KP980785280 * Tgr);
+			 Tgx = FMA(KP195090322, Tgr, KP980785280 * Tgs);
+		    }
+		    {
+			 E Tgu, Tit, Tgy, TiB;
+			 Tgu = Tgq + Tgt;
+			 ri[WS(ios, 34)] = Tgn - Tgu;
+			 ri[WS(ios, 2)] = Tgn + Tgu;
+			 Tit = Tgw + Tgx;
+			 ii[WS(ios, 2)] = Tit + TiA;
+			 ii[WS(ios, 34)] = TiA - Tit;
+			 Tgy = Tgw - Tgx;
+			 ri[WS(ios, 50)] = Tgv - Tgy;
+			 ri[WS(ios, 18)] = Tgv + Tgy;
+			 TiB = Tgt - Tgq;
+			 ii[WS(ios, 18)] = TiB + TiC;
+			 ii[WS(ios, 50)] = TiC - TiB;
+		    }
+	       }
+	       {
+		    E T7V, TjN, TjT, TaH, T8O, TjK, TaK, TjS, TaO, TaU, T9I, TaE, TaR, TaV, TaB;
+		    E TaF, T8N;
+		    T7V = T7x - T7U;
+		    TjN = TjL + TjM;
+		    TjT = TjM - TjL;
+		    TaH = T7x + T7U;
+		    T8N = FMA(KP195090322, T8D, KP980785280 * T8M);
+		    T8O = T8m - T8N;
+		    TjK = T8m + T8N;
+		    {
+			 E TaJ, TaM, TaN, T9u, T9H;
+			 TaJ = FNMS(KP980785280, T8D, KP195090322 * T8M);
+			 TaK = TaI + TaJ;
+			 TjS = TaJ - TaI;
+			 TaM = T96 + T9t;
+			 TaN = T9D + T9G;
+			 TaO = FMA(KP634393284, TaM, KP773010453 * TaN);
+			 TaU = FNMS(KP634393284, TaN, KP773010453 * TaM);
+			 T9u = T96 - T9t;
+			 T9H = T9D - T9G;
+			 T9I = FMA(KP995184726, T9u, KP098017140 * T9H);
+			 TaE = FNMS(KP995184726, T9H, KP098017140 * T9u);
+			 {
+			      E TaP, TaQ, Tan, TaA;
+			      TaP = T9Z + Tam;
+			      TaQ = Taw + Taz;
+			      TaR = FNMS(KP634393284, TaQ, KP773010453 * TaP);
+			      TaV = FMA(KP773010453, TaQ, KP634393284 * TaP);
+			      Tan = T9Z - Tam;
+			      TaA = Taw - Taz;
+			      TaB = FNMS(KP995184726, TaA, KP098017140 * Tan);
+			      TaF = FMA(KP098017140, TaA, KP995184726 * Tan);
+			 }
+		    }
+		    {
+			 E T8P, TaC, TjR, TjU;
+			 T8P = T7V + T8O;
+			 TaC = T9I + TaB;
+			 ri[WS(ios, 47)] = T8P - TaC;
+			 ri[WS(ios, 15)] = T8P + TaC;
+			 TjR = TaE + TaF;
+			 TjU = TjS + TjT;
+			 ii[WS(ios, 15)] = TjR + TjU;
+			 ii[WS(ios, 47)] = TjU - TjR;
+		    }
+		    {
+			 E TaD, TaG, TjV, TjW;
+			 TaD = T7V - T8O;
+			 TaG = TaE - TaF;
+			 ri[WS(ios, 63)] = TaD - TaG;
+			 ri[WS(ios, 31)] = TaD + TaG;
+			 TjV = TaB - T9I;
+			 TjW = TjT - TjS;
+			 ii[WS(ios, 31)] = TjV + TjW;
+			 ii[WS(ios, 63)] = TjW - TjV;
+		    }
+		    {
+			 E TaL, TaS, TjJ, TjO;
+			 TaL = TaH + TaK;
+			 TaS = TaO + TaR;
+			 ri[WS(ios, 39)] = TaL - TaS;
+			 ri[WS(ios, 7)] = TaL + TaS;
+			 TjJ = TaU + TaV;
+			 TjO = TjK + TjN;
+			 ii[WS(ios, 7)] = TjJ + TjO;
+			 ii[WS(ios, 39)] = TjO - TjJ;
+		    }
+		    {
+			 E TaT, TaW, TjP, TjQ;
+			 TaT = TaH - TaK;
+			 TaW = TaU - TaV;
+			 ri[WS(ios, 55)] = TaT - TaW;
+			 ri[WS(ios, 23)] = TaT + TaW;
+			 TjP = TaR - TaO;
+			 TjQ = TjN - TjK;
+			 ii[WS(ios, 23)] = TjP + TjQ;
+			 ii[WS(ios, 55)] = TjQ - TjP;
+		    }
+	       }
+	       {
+		    E TbV, Tjj, Tjp, TcT, Tca, Tjg, TcW, Tjo, Td0, Td6, Tcu, TcQ, Td3, Td7, TcN;
+		    E TcR, Tc9;
+		    TbV = TbN - TbU;
+		    Tjj = Tjh + Tji;
+		    Tjp = Tji - Tjh;
+		    TcT = TbN + TbU;
+		    Tc9 = FMA(KP831469612, Tc5, KP555570233 * Tc8);
+		    Tca = Tc2 - Tc9;
+		    Tjg = Tc2 + Tc9;
+		    {
+			 E TcV, TcY, TcZ, Tcm, Tct;
+			 TcV = FNMS(KP831469612, Tc8, KP555570233 * Tc5);
+			 TcW = TcU + TcV;
+			 Tjo = TcV - TcU;
+			 TcY = Tce + Tcl;
+			 TcZ = Tcp + Tcs;
+			 Td0 = FMA(KP471396736, TcY, KP881921264 * TcZ);
+			 Td6 = FNMS(KP471396736, TcZ, KP881921264 * TcY);
+			 Tcm = Tce - Tcl;
+			 Tct = Tcp - Tcs;
+			 Tcu = FMA(KP956940335, Tcm, KP290284677 * Tct);
+			 TcQ = FNMS(KP956940335, Tct, KP290284677 * Tcm);
+			 {
+			      E Td1, Td2, TcF, TcM;
+			      Td1 = Tcx + TcE;
+			      Td2 = TcI + TcL;
+			      Td3 = FNMS(KP471396736, Td2, KP881921264 * Td1);
+			      Td7 = FMA(KP881921264, Td2, KP471396736 * Td1);
+			      TcF = Tcx - TcE;
+			      TcM = TcI - TcL;
+			      TcN = FNMS(KP956940335, TcM, KP290284677 * TcF);
+			      TcR = FMA(KP290284677, TcM, KP956940335 * TcF);
+			 }
+		    }
+		    {
+			 E Tcb, TcO, Tjn, Tjq;
+			 Tcb = TbV + Tca;
+			 TcO = Tcu + TcN;
+			 ri[WS(ios, 45)] = Tcb - TcO;
+			 ri[WS(ios, 13)] = Tcb + TcO;
+			 Tjn = TcQ + TcR;
+			 Tjq = Tjo + Tjp;
+			 ii[WS(ios, 13)] = Tjn + Tjq;
+			 ii[WS(ios, 45)] = Tjq - Tjn;
+		    }
+		    {
+			 E TcP, TcS, Tjr, Tjs;
+			 TcP = TbV - Tca;
+			 TcS = TcQ - TcR;
+			 ri[WS(ios, 61)] = TcP - TcS;
+			 ri[WS(ios, 29)] = TcP + TcS;
+			 Tjr = TcN - Tcu;
+			 Tjs = Tjp - Tjo;
+			 ii[WS(ios, 29)] = Tjr + Tjs;
+			 ii[WS(ios, 61)] = Tjs - Tjr;
+		    }
+		    {
+			 E TcX, Td4, Tjf, Tjk;
+			 TcX = TcT + TcW;
+			 Td4 = Td0 + Td3;
+			 ri[WS(ios, 37)] = TcX - Td4;
+			 ri[WS(ios, 5)] = TcX + Td4;
+			 Tjf = Td6 + Td7;
+			 Tjk = Tjg + Tjj;
+			 ii[WS(ios, 5)] = Tjf + Tjk;
+			 ii[WS(ios, 37)] = Tjk - Tjf;
+		    }
+		    {
+			 E Td5, Td8, Tjl, Tjm;
+			 Td5 = TcT - TcW;
+			 Td8 = Td6 - Td7;
+			 ri[WS(ios, 53)] = Td5 - Td8;
+			 ri[WS(ios, 21)] = Td5 + Td8;
+			 Tjl = Td3 - Td0;
+			 Tjm = Tjj - Tjg;
+			 ii[WS(ios, 21)] = Tjl + Tjm;
+			 ii[WS(ios, 53)] = Tjm - Tjl;
+		    }
+	       }
+	       {
+		    E Tb1, Tjz, TjF, Tbt, Tb8, Tju, Tbw, TjE, TbA, TbG, Tbg, Tbq, TbD, TbH, Tbn;
+		    E Tbr, Tb7;
+		    Tb1 = TaX - Tb0;
+		    Tjz = Tjv + Tjy;
+		    TjF = Tjy - Tjv;
+		    Tbt = TaX + Tb0;
+		    Tb7 = FMA(KP831469612, Tb5, KP555570233 * Tb6);
+		    Tb8 = Tb4 - Tb7;
+		    Tju = Tb4 + Tb7;
+		    {
+			 E Tbv, Tby, Tbz, Tbc, Tbf;
+			 Tbv = FNMS(KP555570233, Tb5, KP831469612 * Tb6);
+			 Tbw = Tbu + Tbv;
+			 TjE = Tbv - Tbu;
+			 Tby = Tba + Tbb;
+			 Tbz = Tbd + Tbe;
+			 TbA = FMA(KP956940335, Tby, KP290284677 * Tbz);
+			 TbG = FNMS(KP290284677, Tby, KP956940335 * Tbz);
+			 Tbc = Tba - Tbb;
+			 Tbf = Tbd - Tbe;
+			 Tbg = FMA(KP471396736, Tbc, KP881921264 * Tbf);
+			 Tbq = FNMS(KP881921264, Tbc, KP471396736 * Tbf);
+			 {
+			      E TbB, TbC, Tbj, Tbm;
+			      TbB = Tbh + Tbi;
+			      TbC = Tbk + Tbl;
+			      TbD = FNMS(KP290284677, TbC, KP956940335 * TbB);
+			      TbH = FMA(KP290284677, TbB, KP956940335 * TbC);
+			      Tbj = Tbh - Tbi;
+			      Tbm = Tbk - Tbl;
+			      Tbn = FNMS(KP881921264, Tbm, KP471396736 * Tbj);
+			      Tbr = FMA(KP881921264, Tbj, KP471396736 * Tbm);
+			 }
+		    }
+		    {
+			 E Tb9, Tbo, TjD, TjG;
+			 Tb9 = Tb1 + Tb8;
+			 Tbo = Tbg + Tbn;
+			 ri[WS(ios, 43)] = Tb9 - Tbo;
+			 ri[WS(ios, 11)] = Tb9 + Tbo;
+			 TjD = Tbq + Tbr;
+			 TjG = TjE + TjF;
+			 ii[WS(ios, 11)] = TjD + TjG;
+			 ii[WS(ios, 43)] = TjG - TjD;
+		    }
+		    {
+			 E Tbp, Tbs, TjH, TjI;
+			 Tbp = Tb1 - Tb8;
+			 Tbs = Tbq - Tbr;
+			 ri[WS(ios, 59)] = Tbp - Tbs;
+			 ri[WS(ios, 27)] = Tbp + Tbs;
+			 TjH = Tbn - Tbg;
+			 TjI = TjF - TjE;
+			 ii[WS(ios, 27)] = TjH + TjI;
+			 ii[WS(ios, 59)] = TjI - TjH;
+		    }
+		    {
+			 E Tbx, TbE, Tjt, TjA;
+			 Tbx = Tbt + Tbw;
+			 TbE = TbA + TbD;
+			 ri[WS(ios, 35)] = Tbx - TbE;
+			 ri[WS(ios, 3)] = Tbx + TbE;
+			 Tjt = TbG + TbH;
+			 TjA = Tju + Tjz;
+			 ii[WS(ios, 3)] = Tjt + TjA;
+			 ii[WS(ios, 35)] = TjA - Tjt;
+		    }
+		    {
+			 E TbF, TbI, TjB, TjC;
+			 TbF = Tbt - Tbw;
+			 TbI = TbG - TbH;
+			 ri[WS(ios, 51)] = TbF - TbI;
+			 ri[WS(ios, 19)] = TbF + TbI;
+			 TjB = TbD - TbA;
+			 TjC = Tjz - Tju;
+			 ii[WS(ios, 19)] = TjB + TjC;
+			 ii[WS(ios, 51)] = TjC - TjB;
+		    }
+	       }
+	       {
+		    E Tdd, Tj5, Tjb, TdF, Tdk, TiY, TdI, Tja, TdM, TdS, Tds, TdC, TdP, TdT, Tdz;
+		    E TdD, Tdj;
+		    Tdd = Td9 - Tdc;
+		    Tj5 = TiZ + Tj4;
+		    Tjb = Tj4 - TiZ;
+		    TdF = Td9 + Tdc;
+		    Tdj = FMA(KP195090322, Tdh, KP980785280 * Tdi);
+		    Tdk = Tdg - Tdj;
+		    TiY = Tdg + Tdj;
+		    {
+			 E TdH, TdK, TdL, Tdo, Tdr;
+			 TdH = FNMS(KP195090322, Tdi, KP980785280 * Tdh);
+			 TdI = TdG + TdH;
+			 Tja = TdH - TdG;
+			 TdK = Tdm + Tdn;
+			 TdL = Tdp + Tdq;
+			 TdM = FMA(KP995184726, TdK, KP098017140 * TdL);
+			 TdS = FNMS(KP098017140, TdK, KP995184726 * TdL);
+			 Tdo = Tdm - Tdn;
+			 Tdr = Tdp - Tdq;
+			 Tds = FMA(KP634393284, Tdo, KP773010453 * Tdr);
+			 TdC = FNMS(KP773010453, Tdo, KP634393284 * Tdr);
+			 {
+			      E TdN, TdO, Tdv, Tdy;
+			      TdN = Tdt + Tdu;
+			      TdO = Tdw + Tdx;
+			      TdP = FNMS(KP098017140, TdO, KP995184726 * TdN);
+			      TdT = FMA(KP098017140, TdN, KP995184726 * TdO);
+			      Tdv = Tdt - Tdu;
+			      Tdy = Tdw - Tdx;
+			      Tdz = FNMS(KP773010453, Tdy, KP634393284 * Tdv);
+			      TdD = FMA(KP773010453, Tdv, KP634393284 * Tdy);
+			 }
+		    }
+		    {
+			 E Tdl, TdA, Tj9, Tjc;
+			 Tdl = Tdd + Tdk;
+			 TdA = Tds + Tdz;
+			 ri[WS(ios, 41)] = Tdl - TdA;
+			 ri[WS(ios, 9)] = Tdl + TdA;
+			 Tj9 = TdC + TdD;
+			 Tjc = Tja + Tjb;
+			 ii[WS(ios, 9)] = Tj9 + Tjc;
+			 ii[WS(ios, 41)] = Tjc - Tj9;
+		    }
+		    {
+			 E TdB, TdE, Tjd, Tje;
+			 TdB = Tdd - Tdk;
+			 TdE = TdC - TdD;
+			 ri[WS(ios, 57)] = TdB - TdE;
+			 ri[WS(ios, 25)] = TdB + TdE;
+			 Tjd = Tdz - Tds;
+			 Tje = Tjb - Tja;
+			 ii[WS(ios, 25)] = Tjd + Tje;
+			 ii[WS(ios, 57)] = Tje - Tjd;
+		    }
+		    {
+			 E TdJ, TdQ, TiX, Tj6;
+			 TdJ = TdF + TdI;
+			 TdQ = TdM + TdP;
+			 ri[WS(ios, 33)] = TdJ - TdQ;
+			 ri[WS(ios, 1)] = TdJ + TdQ;
+			 TiX = TdS + TdT;
+			 Tj6 = TiY + Tj5;
+			 ii[WS(ios, 1)] = TiX + Tj6;
+			 ii[WS(ios, 33)] = Tj6 - TiX;
+		    }
+		    {
+			 E TdR, TdU, Tj7, Tj8;
+			 TdR = TdF - TdI;
+			 TdU = TdS - TdT;
+			 ri[WS(ios, 49)] = TdR - TdU;
+			 ri[WS(ios, 17)] = TdR + TdU;
+			 Tj7 = TdP - TdM;
+			 Tj8 = Tj5 - TiY;
+			 ii[WS(ios, 17)] = Tj7 + Tj8;
+			 ii[WS(ios, 49)] = Tj8 - Tj7;
+		    }
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_COS, 0, 1},
+     {TW_SIN, 0, 1},
+     {TW_COS, 0, 3},
+     {TW_SIN, 0, 3},
+     {TW_COS, 0, 9},
+     {TW_SIN, 0, 9},
+     {TW_COS, 0, 27},
+     {TW_SIN, 0, 27},
+     {TW_COS, 0, 63},
+     {TW_SIN, 0, 63},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 64, "t2_64", twinstr, {880, 386, 274, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t2_64) (planner *p) {
+     X(kdft_dit_register) (p, t2_64, &desc);
+}
diff --git a/src/fftw3/dft/codelets/standard/t2_8.c b/src/fftw3/dft/codelets/standard/t2_8.c
new file mode 100644
index 0000000..d9aec9a
--- /dev/null
+++ b/src/fftw3/dft/codelets/standard/t2_8.c
@@ -0,0 +1,192 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:30:08 EDT 2003 */
+
+#include "codelet-dft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_twiddle -compact -variables 4 -twiddle-log3 -n 8 -name t2_8 -include t.h */
+
+/*
+ * This function contains 74 FP additions, 44 FP multiplications,
+ * (or, 56 additions, 26 multiplications, 18 fused multiply/add),
+ * 50 stack variables, and 32 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: t2_8.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t2_8.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ * $Id: t2_8.c,v 1.1 2008/10/17 06:11:09 scuri Exp $
+ */
+
+#include "t.h"
+
+static const R *t2_8(R *ri, R *ii, const R *W, stride ios, int m, int dist)
+{
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m; i > 0; i = i - 1, ri = ri + dist, ii = ii + dist, W = W + 6) {
+	  E T1, T1c, TJ, T17, TY, TV, TR, Tk, Tr, TN, TM, Tw, TB, TS, Te;
+	  E T1b;
+	  T1 = ri[0];
+	  T1c = ii[0];
+	  {
+	       E T9, Td, Th, Tj, To, Tq, Ty, TA, Tv, Tu, T3, T6, T4, Tb, T7;
+	       E Ta, Tg, Ti, TI, TX, Tl, Tm, Tn, TW, Tp, TF;
+	       T9 = ri[WS(ios, 4)];
+	       Td = ii[WS(ios, 4)];
+	       Th = ri[WS(ios, 2)];
+	       Tj = ii[WS(ios, 2)];
+	       To = ri[WS(ios, 6)];
+	       Tq = ii[WS(ios, 6)];
+	       Ty = ri[WS(ios, 5)];
+	       TA = ii[WS(ios, 5)];
+	       Tv = ii[WS(ios, 1)];
+	       Tu = ri[WS(ios, 1)];
+	       {
+		    E TD, TE, TG, TH, T2, T5;
+		    TD = ri[WS(ios, 7)];
+		    TE = ii[WS(ios, 7)];
+		    TG = ri[WS(ios, 3)];
+		    TH = ii[WS(ios, 3)];
+		    T2 = W[2];
+		    T5 = W[3];
+		    T3 = W[0];
+		    T6 = W[1];
+		    T4 = T2 * T3;
+		    Tb = T5 * T3;
+		    T7 = T5 * T6;
+		    Ta = T2 * T6;
+		    Tg = T4 + T7;
+		    Ti = Ta - Tb;
+		    TI = FMA(T2, TG, T5 * TH);
+		    TX = FNMS(T5, TG, T2 * TH);
+		    Tl = W[4];
+		    Tm = W[5];
+		    Tn = FMA(Tl, T3, Tm * T6);
+		    TW = FNMS(Tm, TD, Tl * TE);
+		    Tp = FNMS(Tm, T3, Tl * T6);
+		    TF = FMA(Tl, TD, Tm * TE);
+	       }
+	       TJ = TF + TI;
+	       T17 = TW + TX;
+	       TY = TW - TX;
+	       TV = TF - TI;
+	       TR = FNMS(T6, Tu, T3 * Tv);
+	       Tk = FNMS(Ti, Tj, Tg * Th);
+	       Tr = FNMS(Tp, Tq, Tn * To);
+	       TN = FMA(Tp, To, Tn * Tq);
+	       TM = FMA(Ti, Th, Tg * Tj);
+	       Tw = FMA(T3, Tu, T6 * Tv);
+	       {
+		    E Tx, Tz, T8, Tc;
+		    Tx = FNMS(Tm, Ti, Tl * Tg);
+		    Tz = FMA(Tl, Ti, Tm * Tg);
+		    TB = FMA(Tx, Ty, Tz * TA);
+		    TS = FNMS(Tz, Ty, Tx * TA);
+		    T8 = T4 - T7;
+		    Tc = Ta + Tb;
+		    Te = FMA(T8, T9, Tc * Td);
+		    T1b = FNMS(Tc, T9, T8 * Td);
+	       }
+	  }
+	  {
+	       E TK, T1f, T18, T19, Tt, T15, T1e, T1g, TC, T16;
+	       TC = Tw + TB;
+	       TK = TC + TJ;
+	       T1f = TJ - TC;
+	       T16 = TR + TS;
+	       T18 = T16 - T17;
+	       T19 = T16 + T17;
+	       {
+		    E Tf, Ts, T1a, T1d;
+		    Tf = T1 + Te;
+		    Ts = Tk + Tr;
+		    Tt = Tf + Ts;
+		    T15 = Tf - Ts;
+		    T1a = TM + TN;
+		    T1d = T1b + T1c;
+		    T1e = T1a + T1d;
+		    T1g = T1d - T1a;
+	       }
+	       ri[WS(ios, 4)] = Tt - TK;
+	       ii[WS(ios, 4)] = T1e - T19;
+	       ri[0] = Tt + TK;
+	       ii[0] = T19 + T1e;
+	       ri[WS(ios, 6)] = T15 - T18;
+	       ii[WS(ios, 6)] = T1g - T1f;
+	       ri[WS(ios, 2)] = T15 + T18;
+	       ii[WS(ios, 2)] = T1f + T1g;
+	  }
+	  {
+	       E TZ, T13, TP, T11, TU, T12, T1k, T1m, TL, TO;
+	       TZ = TV - TY;
+	       T13 = TV + TY;
+	       TL = T1 - Te;
+	       TO = TM - TN;
+	       TP = TL + TO;
+	       T11 = TL - TO;
+	       {
+		    E TQ, TT, T1i, T1j;
+		    TQ = Tw - TB;
+		    TT = TR - TS;
+		    TU = TQ + TT;
+		    T12 = TT - TQ;
+		    T1i = T1c - T1b;
+		    T1j = Tk - Tr;
+		    T1k = T1i - T1j;
+		    T1m = T1j + T1i;
+	       }
+	       {
+		    E T10, T1h, T14, T1l;
+		    T10 = KP707106781 * (TU + TZ);
+		    ri[WS(ios, 5)] = TP - T10;
+		    ri[WS(ios, 1)] = TP + T10;
+		    T1h = KP707106781 * (T12 + T13);
+		    ii[WS(ios, 1)] = T1h + T1k;
+		    ii[WS(ios, 5)] = T1k - T1h;
+		    T14 = KP707106781 * (T12 - T13);
+		    ri[WS(ios, 7)] = T11 - T14;
+		    ri[WS(ios, 3)] = T11 + T14;
+		    T1l = KP707106781 * (TZ - TU);
+		    ii[WS(ios, 3)] = T1l + T1m;
+		    ii[WS(ios, 7)] = T1m - T1l;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_COS, 0, 1},
+     {TW_SIN, 0, 1},
+     {TW_COS, 0, 3},
+     {TW_SIN, 0, 3},
+     {TW_COS, 0, 7},
+     {TW_SIN, 0, 7},
+     {TW_NEXT, 1, 0}
+};
+
+static const ct_desc desc = { 8, "t2_8", twinstr, {56, 26, 18, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_t2_8) (planner *p) {
+     X(kdft_dit_register) (p, t2_8, &desc);
+}
diff --git a/src/fftw3/dft/codelets/t.c b/src/fftw3/dft/codelets/t.c
new file mode 100644
index 0000000..b3fe55b
--- /dev/null
+++ b/src/fftw3/dft/codelets/t.c
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "codelet-dft.h"
+#include "t.h"
+
+static int okp(const ct_desc *d,
+	       const R *rio, const R *iio, 
+	       int ios, int vs, int m, int dist, const planner *plnr)
+{
+     UNUSED(rio); UNUSED(iio); UNUSED(m); UNUSED(plnr);
+     return (1
+	     && (!d->s1 || (d->s1 == ios))
+	     && (!d->s2 || (d->s2 == vs))
+	     && (!d->dist || (d->dist == dist))
+	  );
+}
+
+const ct_genus GENUS = { okp, 1 };
diff --git a/src/fftw3/dft/codelets/t.h b/src/fftw3/dft/codelets/t.h
new file mode 100644
index 0000000..ecb5abc
--- /dev/null
+++ b/src/fftw3/dft/codelets/t.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+
+#define GENUS X(dft_t_genus)
+extern const ct_genus GENUS;
diff --git a/src/fftw3/dft/conf.c b/src/fftw3/dft/conf.c
new file mode 100644
index 0000000..30edb41
--- /dev/null
+++ b/src/fftw3/dft/conf.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: conf.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+#include "dft.h"
+
+static const solvtab s =
+{
+     SOLVTAB(X(dft_indirect_register)),
+     SOLVTAB(X(dft_rank0_register)),
+     SOLVTAB(X(dft_rank_geq2_register)),
+     SOLVTAB(X(dft_vrank_geq1_register)),
+     SOLVTAB(X(dft_vrank2_transpose_register)),
+     SOLVTAB(X(dft_vrank3_transpose_register)),
+     SOLVTAB(X(dft_buffered_register)),
+     SOLVTAB(X(dft_generic_register)),
+     SOLVTAB(X(dft_rader_register)),
+     SOLVTAB(X(dft_nop_register)),
+     SOLVTAB_END
+};
+
+void X(dft_conf_standard)(planner *p)
+{
+     X(solvtab_exec)(s, p);
+     X(solvtab_exec)(X(solvtab_dft_standard), p);
+     X(solvtab_exec)(X(solvtab_dft_inplace), p);
+#if HAVE_K7
+     X(solvtab_exec)(X(solvtab_dft_k7), p);
+#endif
+#if HAVE_SIMD
+     X(solvtab_exec)(X(solvtab_dft_simd), p);
+#endif
+}
diff --git a/src/fftw3/dft/ct-dif.c b/src/fftw3/dft/ct-dif.c
new file mode 100644
index 0000000..490dfc9
--- /dev/null
+++ b/src/fftw3/dft/ct-dif.c
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: ct-dif.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* decimation in time Cooley-Tukey */
+#include "dft.h"
+#include "ct.h"
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const plan_ct *ego = (const plan_ct *) ego_;
+
+     {
+          int i, m = ego->m, vl = ego->vl;
+          int is = ego->is, ivs = ego->ivs;
+
+          for (i = 0; i < vl; ++i)
+               ego->k.dif(ri + i * ivs, ii + i * ivs, ego->td->W,
+			  ego->ios, m, is);
+     }
+
+     /* two-dimensional r x vl sub-transform: */
+     {
+	  plan *cld0 = ego->cld;
+	  plan_dft *cld = (plan_dft *) cld0;
+	  cld->apply(cld0, ri, ii, ro, io);
+     }
+}
+
+static int applicable0(const solver_ct *ego, const problem *p_,
+		       const planner *plnr)
+{
+     if (X(dft_ct_applicable)(ego, p_)) {
+	  int ivs, ovs;
+	  int vl;
+          const ct_desc *e = ego->desc;
+          const problem_dft *p = (const problem_dft *) p_;
+          iodim *d = p->sz->dims;
+	  int m = d[0].n / e->radix;
+	  X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs);
+          return (1
+                  /* DIF destroys the input and we don't like it */
+                  && (p->ri == p->ro || DESTROY_INPUTP(plnr))
+
+		  && (e->genus->okp(e, p->ri, p->ii,
+				    (int)m * d[0].is, 0, m, d[0].is, plnr))
+		  && (e->genus->okp(e, p->ri + ivs, p->ii + ivs,
+				    (int)m * d[0].is, 0, m, d[0].is, plnr))
+	       );
+     }
+     return 0;
+}
+
+static int applicable(const solver_ct *ego, const problem *p_,
+		 const planner *plnr)
+{
+     const problem_dft *p;
+     
+     if (!applicable0(ego, p_, plnr))  return 0;
+
+     p = (const problem_dft *) p_;
+
+     /* emulate fftw2 behavior */
+     if (NO_VRECURSEP(plnr) && (p->vecsz->rnk > 0)) return 0;
+
+     if (NO_UGLYP(plnr) && X(ct_uglyp)(16, p->sz->dims[0].n, ego->desc->radix))
+	  return 0;
+
+     return 1;
+}
+
+static void finish(plan_ct *ego)
+{
+     const ct_desc *d = ego->slv->desc;
+     ego->ios = X(mkstride)(ego->r, ego->m * ego->is);
+     X(ops_madd)(ego->vl * ego->m / d->genus->vl, &d->ops, &ego->cld->ops,
+		 &ego->super.super.ops);
+}
+
+static plan *mkplan(const solver *ego, const problem *p, planner *plnr)
+{
+     static const ctadt adt = {
+	  sizeof(plan_ct), X(dft_mkcld_dif), finish, applicable, apply
+     };
+     return X(mkplan_dft_ct)((const solver_ct *) ego, p, plnr, &adt);
+}
+
+
+solver *X(mksolver_dft_ct_dif)(kdft_dif codelet, const ct_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     static const char name[] = "dft-dif";
+     union kct k;
+     k.dif = codelet;
+
+     return X(mksolver_dft_ct)(k, desc, name, &sadt);
+}
diff --git a/src/fftw3/dft/ct-dit.c b/src/fftw3/dft/ct-dit.c
new file mode 100644
index 0000000..3251c20
--- /dev/null
+++ b/src/fftw3/dft/ct-dit.c
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: ct-dit.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* decimation in time Cooley-Tukey */
+#include "dft.h"
+#include "ct.h"
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const plan_ct *ego = (const plan_ct *) ego_;
+     plan *cld0 = ego->cld;
+     plan_dft *cld = (plan_dft *) cld0;
+
+     /* two-dimensional r x vl sub-transform: */
+     cld->apply(cld0, ri, ii, ro, io);
+
+     {
+          int i, m = ego->m, vl = ego->vl;
+          int os = ego->os, ovs = ego->ovs;
+
+          for (i = 0; i < vl; ++i)
+               ego->k.dit(ro + i * ovs, io + i * ovs, ego->td->W,
+			  ego->ios, m, os);
+     }
+}
+
+static int applicable0(const solver_ct *ego, const problem *p_,
+		       const planner *plnr)
+{
+     UNUSED(plnr);
+     if (X(dft_ct_applicable)(ego, p_)) {
+	  int ivs, ovs;
+	  int vl;
+          const ct_desc *e = ego->desc;
+          const problem_dft *p = (const problem_dft *) p_;
+          iodim *d = p->sz->dims;
+	  int m = d[0].n / e->radix;
+	  X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs);
+          return (1
+		  && (e->genus->okp(e, p->ro, p->io, 
+				    (int)m * d[0].os, 0, m, d[0].os, plnr))
+		  && (e->genus->okp(e, p->ro + ovs, p->io + ovs, 
+				    (int)m * d[0].os, 0, m, d[0].os, plnr))
+	       );
+     }
+     return 0;
+}
+
+static int applicable(const solver_ct *ego, const problem *p_,
+		      const planner *plnr)
+{
+     const problem_dft *p;
+
+     if (!applicable0(ego, p_, plnr))
+          return 0;
+
+     p = (const problem_dft *) p_;
+
+     /* emulate fftw2 behavior */
+     if (NO_VRECURSEP(plnr) && (p->vecsz->rnk > 0))  return 0;
+
+     if (NO_UGLYP(plnr)) {
+	  if (X(ct_uglyp)(16, p->sz->dims[0].n, ego->desc->radix)) return 0;
+	  if (NONTHREADED_ICKYP(plnr))
+	       return 0; /* prefer threaded version */
+     }
+
+     return 1;
+}
+
+
+static void finish(plan_ct *ego)
+{
+     const ct_desc *d = ego->slv->desc;
+     ego->ios = X(mkstride)(ego->r, ego->m * ego->os);
+     X(ops_madd)(ego->vl * ego->m / d->genus->vl, &d->ops, &ego->cld->ops,
+		 &ego->super.super.ops);
+}
+
+static plan *mkplan(const solver *ego, const problem *p, planner *plnr)
+{
+     static const ctadt adt = {
+	  sizeof(plan_ct), X(dft_mkcld_dit), finish, applicable, apply
+     };
+     return X(mkplan_dft_ct)((const solver_ct *) ego, p, plnr, &adt);
+}
+
+
+solver *X(mksolver_dft_ct_dit)(kdft_dit codelet, const ct_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     static const char name[] = "dft-dit";
+     union kct k;
+     k.dit = codelet;
+
+     return X(mksolver_dft_ct)(k, desc, name, &sadt);
+}
diff --git a/src/fftw3/dft/ct-ditbuf.c b/src/fftw3/dft/ct-ditbuf.c
new file mode 100644
index 0000000..2f7b57f
--- /dev/null
+++ b/src/fftw3/dft/ct-ditbuf.c
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: ct-ditbuf.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* decimation in time Cooley-Tukey.  Codelet operates on
+   contiguous buffer rather than directly on the output array.  */
+
+/* FIXME: find a way to use rank-0 transforms for this stuff */
+
+#include "dft.h"
+#include "ct.h"
+
+/*
+   Copy A -> B, where A and B are n0 x n1 complex matrices
+   such that the (i0, i1) element has index (i0 * s0 + i1 * s1). 
+*/
+static void cpy(int n0, int n1, 
+		const R *rA, const R *iA, int sa0, int sa1, 
+		R *rB, R *iB, int sb0, int sb1)
+{
+     int i0, i1;
+     int ima = iA - rA, imb = iB - rB;
+
+     for (i0 = 0; i0 < n0; ++i0) {
+	  const R *pa; 
+	  R *pb;
+
+	  pa = rA; rA += sa0;
+	  pb = rB; rB += sb0;
+	  for (i1 = 0; i1 < n1; ++i1) {
+	       R xr = pa[0], xi = pa[ima];
+	       pb[0] = xr; pb[imb] = xi; 
+	       pa += sa1; pb += sb1;
+	  }
+     }
+}
+
+static const R *doit(kdft_dit k, R *rA, R *iA, const R *W, int ios, int dist, 
+		     int r, int batchsz, R *buf, stride bufstride)
+{
+     cpy(r, batchsz, rA, iA, ios, dist, buf, buf + 1, 2, 2 * r);
+     W = k(buf, buf + 1, W, bufstride, batchsz, 2 * r);
+     cpy(r, batchsz, buf, buf + 1, 2, 2 * r, rA, iA, ios, dist);
+     return W;
+}
+
+#define BATCHSZ 4 /* FIXME: parametrize? */
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const plan_ct *ego = (const plan_ct *) ego_;
+     plan *cld0 = ego->cld;
+     plan_dft *cld = (plan_dft *) cld0;
+
+     /* two-dimensional r x vl sub-transform: */
+     cld->apply(cld0, ri, ii, ro, io);
+
+     {
+          int i, j, m = ego->m, vl = ego->vl, r = ego->r;
+          int os = ego->os, ovs = ego->ovs, ios = ego->iios;
+	  R *buf;
+
+	  STACK_MALLOC(R *, buf, r * BATCHSZ * 2 * sizeof(R));
+
+          for (i = 0; i < vl; ++i) {
+	       R *rA = ro + i * ovs, *iA = io + i * ovs;
+	       const R *W = ego->td->W;
+
+	       for (j = m; j >= BATCHSZ; j -= BATCHSZ) {
+		    W = doit(ego->k.dit, rA, iA, W, ios, os, r, 
+			     BATCHSZ, buf, ego->vs);
+		    rA += os * (int)BATCHSZ;
+		    iA += os * (int)BATCHSZ;
+	       }
+
+	       /* do remaining j calls, if any */
+	       if (j > 0)
+		    doit(ego->k.dit, rA, iA, W, ios, os, r, j, buf, ego->vs);
+
+	  }
+
+	  STACK_FREE(buf);
+     }
+}
+
+static int applicable0(const solver_ct *ego, const problem *p_,
+		       const planner *plnr)
+{
+     UNUSED(plnr);
+     if (X(dft_ct_applicable)(ego, p_)) {
+          const ct_desc *e = ego->desc;
+          const problem_dft *p = (const problem_dft *) p_;
+          iodim *d = p->sz->dims;
+	  int m = d[0].n / e->radix;
+          return (1
+
+                  /* check both batch size and remainder */
+		  && (m < BATCHSZ ||
+		      (e->genus->okp(e, 0, ((const R *)0)+1, 2, 0, BATCHSZ,
+				     2 * e->radix, plnr)))
+		  && (e->genus->okp(e, 0, ((const R *)0)+1, 2, 0, m % BATCHSZ,
+				    2 * e->radix, plnr))
+	       );
+     }
+     return 0;
+}
+
+static int applicable(const solver_ct *ego, const problem *p_,
+		      const planner *plnr)
+{
+     const problem_dft *p;
+
+     if (!applicable0(ego, p_, plnr)) return 0;
+
+     p = (const problem_dft *) p_;
+
+     /* emulate fftw2 behavior */
+     if (NO_VRECURSEP(plnr) && (p->vecsz->rnk > 0))  return 0;
+
+     if (NO_UGLYP(plnr) && X(ct_uglyp)(512, p->sz->dims[0].n,
+				       ego->desc->radix))
+	  return 0;
+
+     return 1;
+}
+
+
+static void finish(plan_ct *ego)
+{
+     const ct_desc *d = ego->slv->desc;
+     ego->iios = ego->m * ego->os;
+     ego->vs = X(mkstride)(ego->r, 2);
+     X(ops_madd)(ego->vl * ego->m / d->genus->vl, &d->ops, &ego->cld->ops,
+		 &ego->super.super.ops);
+
+     /* 4 load/stores * N * VL */
+     ego->super.super.ops.other += 4 * ego->r * ego->m * ego->vl;
+}
+
+static plan *mkplan(const solver *ego, const problem *p, planner *plnr)
+{
+     static const ctadt adt = {
+	  sizeof(plan_ct), X(dft_mkcld_dit), finish, applicable, apply
+     };
+     return X(mkplan_dft_ct)((const solver_ct *) ego, p, plnr, &adt);
+}
+
+
+solver *X(mksolver_dft_ct_ditbuf)(kdft_dit codelet, const ct_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     static const char name[] = "dft-ditbuf";
+     union kct k;
+     k.dit = codelet;
+
+     return X(mksolver_dft_ct)(k, desc, name, &sadt);
+}
diff --git a/src/fftw3/dft/ct-ditf.c b/src/fftw3/dft/ct-ditf.c
new file mode 100644
index 0000000..10823ab
--- /dev/null
+++ b/src/fftw3/dft/ct-ditf.c
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: ct-ditf.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* decimation in time Cooley-Tukey */
+#include "dft.h"
+#include "ct.h"
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const plan_ct *ego = (const plan_ct *) ego_;
+     plan *cld0 = ego->cld;
+     plan_dft *cld = (plan_dft *) cld0;
+
+     UNUSED(ro);  /* == ri */
+     UNUSED(io);  /* == ii */
+     ego->k.difsq(ri, ii, ego->td->W, ego->ios, ego->vs, ego->m, ego->is);
+
+     /* two-dimensional r x vl sub-transform: */
+     cld->apply(cld0, ri, ii, ri, ii);
+}
+
+static int applicable(const solver_ct *ego, const problem *p_,
+		      const planner *plnr)
+{
+     UNUSED(plnr);
+     if (X(dft_ct_applicable)(ego, p_)) {
+          const ct_desc *e = ego->desc;
+          const problem_dft *p = (const problem_dft *) p_;
+          iodim *d = p->sz->dims, *vd = p->vecsz->dims;
+	  int m = d[0].n / e->radix;
+
+          return (1
+                  && p->ri == p->ro  /* inplace only */
+                  && p->vecsz->rnk == 1
+                  && vd[0].n == e->radix
+                  && d[0].os == vd[0].is
+                  && d[0].is == (int)e->radix * vd[0].is
+                  && vd[0].os == (int)d[0].n * vd[0].is
+
+		  && (e->genus->okp(e, p->ri, p->ii, 
+				    vd[0].os, vd[0].is, m, d[0].is, plnr))
+	       );
+     }
+     return 0;
+}
+
+static void finish(plan_ct *ego)
+{
+     const ct_desc *d = ego->slv->desc;
+     ego->ios = X(mkstride)(ego->r, ego->ovs);
+     ego->vs = X(mkstride)(ego->r, ego->ivs);
+     X(ops_madd)(ego->m / d->genus->vl, &ego->slv->desc->ops,
+		 &ego->cld->ops, &ego->super.super.ops);
+}
+
+static problem *mkcld(const solver_ct *ego, const problem_dft *p)
+{
+     iodim *d = p->sz->dims;
+     iodim *vd = p->vecsz->dims;
+     const ct_desc *e = ego->desc;
+
+     return X(mkproblem_dft_d)(
+	  X(mktensor_1d)(d[0].n / e->radix, d[0].is, d[0].is),
+	  X(mktensor_2d)(vd[0].n, vd[0].os, vd[0].os,
+			 e->radix, vd[0].is,vd[0].is),
+	  p->ro, p->io, p->ro, p->io);
+}
+
+static plan *mkplan(const solver *ego, const problem *p, planner *plnr)
+{
+     static const ctadt adt = {
+	  sizeof(plan_ct), mkcld, finish, applicable, apply
+     };
+     return X(mkplan_dft_ct)((const solver_ct *) ego, p, plnr, &adt);
+}
+
+
+solver *X(mksolver_dft_ct_ditf)(kdft_difsq codelet, const ct_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     static const char name[] = "dft-ditf";
+     union kct k;
+     k.difsq = codelet;
+
+     return X(mksolver_dft_ct)(k, desc, name, &sadt);
+}
diff --git a/src/fftw3/dft/ct.c b/src/fftw3/dft/ct.c
new file mode 100644
index 0000000..1cee970
--- /dev/null
+++ b/src/fftw3/dft/ct.c
@@ -0,0 +1,170 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: ct.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* generic Cooley-Tukey routines */
+#include "dft.h"
+#include "ct.h"
+
+static void destroy(plan *ego_)
+{
+     plan_ct *ego = (plan_ct *) ego_;
+
+     X(plan_destroy_internal)(ego->cld);
+     X(stride_destroy)(ego->ios);
+     X(stride_destroy)(ego->vs);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     plan_ct *ego = (plan_ct *) ego_;
+     plan *cld = ego->cld;
+
+     AWAKE(cld, flg);
+     X(twiddle_awake)(flg, &ego->td, ego->slv->desc->tw, 
+		      ego->r * ego->m, ego->r, ego->m);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const plan_ct *ego = (const plan_ct *) ego_;
+     const solver_ct *slv = ego->slv;
+     const ct_desc *e = slv->desc;
+
+     p->print(p, "(%s-%d/%d%v \"%s\"%(%p%))",
+              slv->nam, ego->r, X(twiddle_length)(ego->r, e->tw),
+	      ego->vl, e->nam, ego->cld);
+}
+
+#define divides(a, b) (((int)(b) % (int)(a)) == 0)
+
+int X(dft_ct_applicable)(const solver_ct *ego, const problem *p_)
+{
+     if (DFTP(p_)) {
+          const problem_dft *p = (const problem_dft *) p_;
+          const ct_desc *d = ego->desc;
+          return (1
+                  && p->sz->rnk == 1
+                  && p->vecsz->rnk <= 1
+                  && divides(d->radix, p->sz->dims[0].n)
+	       );
+     }
+     return 0;
+}
+
+
+static const plan_adt padt =
+{
+     X(dft_solve),
+     awake,
+     print,
+     destroy
+};
+
+
+plan *X(mkplan_dft_ct)(const solver_ct *ego,
+                       const problem *p_,
+                       planner *plnr,
+                       const ctadt *adt)
+{
+     plan_ct *pln;
+     plan *cld;
+     int n, r, m;
+     iodim *d;
+     const problem_dft *p;
+     const ct_desc *e = ego->desc;
+
+     if (!adt->applicable(ego, p_, plnr))
+          return (plan *) 0;
+
+     p = (const problem_dft *) p_;
+     d = p->sz->dims;
+     n = d[0].n;
+     r = e->radix;
+     m = n / r;
+
+     cld = X(mkplan_d)(plnr, adt->mkcld(ego, p));
+
+     if (!cld)
+          return (plan *) 0;
+
+     A(adt->pln_size >= sizeof(plan_ct));
+     pln = (plan_ct *) X(mkplan_dft)(adt->pln_size, &padt, adt->apply);
+
+     pln->slv = ego;
+     pln->cld = cld;
+     pln->k = ego->k;
+     pln->r = r;
+     pln->m = m;
+
+     pln->is = d[0].is;
+     pln->os = d[0].os;
+
+     pln->ios = pln->vs = 0;
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+
+     pln->td = 0;
+     adt->finish(pln);
+
+     return &(pln->super.super);
+}
+
+solver *X(mksolver_dft_ct)(union kct k, const ct_desc *desc,
+                           const char *nam, const solver_adt *adt)
+{
+     solver_ct *slv;
+
+     slv = MKSOLVER(solver_ct, adt);
+
+     slv->desc = desc;
+     slv->k = k;
+     slv->nam = nam;
+     return &(slv->super);
+}
+
+/* routines to create children are shared by many solvers */
+problem *X(dft_mkcld_dit)(const solver_ct *ego, const problem_dft *p)
+{
+     iodim *d = p->sz->dims;
+     const ct_desc *e = ego->desc;
+     int m = d[0].n / e->radix;
+
+     tensor *radix = X(mktensor_1d)(e->radix, d[0].is, m * d[0].os);
+     tensor *cld_vec = X(tensor_append)(radix, p->vecsz);
+     X(tensor_destroy)(radix);
+
+     return X(mkproblem_dft_d)(X(mktensor_1d)(m, e->radix * d[0].is, d[0].os),
+			       cld_vec, p->ri, p->ii, p->ro, p->io);
+}
+
+problem *X(dft_mkcld_dif)(const solver_ct *ego, const problem_dft *p)
+{
+     iodim *d = p->sz->dims;
+     const ct_desc *e = ego->desc;
+     int m = d[0].n / e->radix;
+
+     tensor *radix = X(mktensor_1d)(e->radix, m * d[0].is, d[0].os);
+     tensor *cld_vec = X(tensor_append)(radix, p->vecsz);
+     X(tensor_destroy)(radix);
+
+     return X(mkproblem_dft_d)(X(mktensor_1d)(m, d[0].is, e->radix * d[0].os),
+			       cld_vec, p->ri, p->ii, p->ro, p->io);
+}
diff --git a/src/fftw3/dft/ct.h b/src/fftw3/dft/ct.h
new file mode 100644
index 0000000..7ee8ff7
--- /dev/null
+++ b/src/fftw3/dft/ct.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: ct.h,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* Cooley-Tukey variants */
+union kct {
+     kdft_dit dit;
+     kdft_dif dif;
+     kdft_difsq difsq;
+};
+
+typedef struct {
+     solver super;
+     const char *nam;
+     const ct_desc *desc;
+     union kct k;
+} solver_ct;
+
+typedef struct {
+     plan_dft super;
+     union kct k;
+     plan *cld;
+     twid *td;
+     int r, m, vl;
+     int is, os, ivs, ovs, iios;
+     stride ios, vs;
+     const solver_ct *slv;
+} plan_ct;
+
+/* data type describing a generic Cooley-Tukey solver */
+typedef struct
+{
+     size_t pln_size;
+     problem *(*mkcld)(const solver_ct *, const problem_dft *p);
+     void (*finish)(plan_ct *ego);
+     int (*applicable)(const solver_ct *ego, const problem *p,
+		       const planner *plnr);
+     dftapply apply;
+} ctadt;
+
+int X(dft_ct_applicable)(const solver_ct *ego, const problem *p_);
+
+plan *X(mkplan_dft_ct)(const solver_ct *ego,
+                       const problem *p_,
+                       planner *plnr,
+                       const ctadt *adt);
+
+solver *X(mksolver_dft_ct)(union kct k, const ct_desc *desc,
+                           const char *nam, const solver_adt *adt);
+
+problem *X(dft_mkcld_dit)(const solver_ct *ego, const problem_dft *p);
+problem *X(dft_mkcld_dif)(const solver_ct *ego, const problem_dft *p);
diff --git a/src/fftw3/dft/dft.h b/src/fftw3/dft/dft.h
new file mode 100644
index 0000000..cc88a1d
--- /dev/null
+++ b/src/fftw3/dft/dft.h
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: dft.h,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+#ifndef __DFT_H__
+#define __DFT_H__
+
+#include "ifftw.h"
+#include "codelet-dft.h"
+
+/* problem.c: */
+typedef struct {
+     problem super;
+     tensor *sz, *vecsz;
+     R *ri, *ii, *ro, *io;
+} problem_dft;
+
+int X(problem_dft_p)(const problem *p);
+#define DFTP X(problem_dft_p)  /* shorthand */
+
+void X(dft_zerotens)(tensor *sz, R *ri, R *ii);
+problem *X(mkproblem_dft)(const tensor *sz, const tensor *vecsz,
+                          R *ri, R *ii, R *ro, R *io);
+problem *X(mkproblem_dft_d)(tensor *sz, tensor *vecsz,
+                            R *ri, R *ii, R *ro, R *io);
+
+/* solve.c: */
+void X(dft_solve)(const plan *ego_, const problem *p_);
+
+/* plan.c: */
+typedef void (*dftapply) (const plan *ego, R *ri, R *ii, R *ro, R *io);
+
+typedef struct {
+     plan super;
+     dftapply apply;
+} plan_dft;
+
+plan *X(mkplan_dft)(size_t size, const plan_adt *adt, dftapply apply);
+
+#define MKPLAN_DFT(type, adt, apply) \
+  (type *)X(mkplan_dft)(sizeof(type), adt, apply)
+
+/* various solvers */
+solver *X(mksolver_dft_direct)(kdft k, const kdft_desc *desc);
+solver *X(mksolver_dft_ct_dit)(kdft_dit codelet, const ct_desc *desc);
+solver *X(mksolver_dft_ct_ditbuf)(kdft_dit codelet, const ct_desc *desc);
+solver *X(mksolver_dft_ct_dif)(kdft_dif codelet, const ct_desc *desc);
+solver *X(mksolver_dft_ct_ditf)(kdft_difsq codelet, const ct_desc *desc);
+
+extern void (*X(kdft_dit_register_hook))(planner *, kdft_dit, const ct_desc *);
+extern void (*X(kdft_dif_register_hook))(planner *, kdft_dif, const ct_desc *);
+
+void X(dft_rank0_register)(planner *p);
+void X(dft_rank_geq2_register)(planner *p);
+void X(dft_indirect_register)(planner *p);
+void X(dft_vrank_geq1_register)(planner *p);
+void X(dft_vrank2_transpose_register)(planner *p);
+void X(dft_vrank3_transpose_register)(planner *p);
+void X(dft_buffered_register)(planner *p);
+void X(dft_generic_register)(planner *p);
+void X(dft_rader_register)(planner *p);
+void X(dft_nop_register)(planner *p);
+
+/* rader-omega.c: auxiliary stuff for rader */
+R *X(dft_rader_mkomega)(plan *p_, int n, int ginv);
+void X(dft_rader_free_omega)(R **omega);
+
+/* configurations */
+void X(dft_conf_standard)(planner *p);
+
+#endif /* __DFT_H__ */
diff --git a/src/fftw3/dft/direct.c b/src/fftw3/dft/direct.c
new file mode 100644
index 0000000..cb11c88
--- /dev/null
+++ b/src/fftw3/dft/direct.c
@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: direct.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* direct DFT solver, if we have a codelet */
+
+#include "dft.h"
+
+typedef struct {
+     solver super;
+     const kdft_desc *desc;
+     kdft k;
+} S;
+
+typedef struct {
+     plan_dft super;
+
+     stride is, os;
+     int vl;
+     int ivs, ovs;
+     kdft k;
+     const S *slv;
+} P;
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     ASSERT_ALIGNED_DOUBLE;
+     ego->k(ri, ii, ro, io, ego->is, ego->os, ego->vl, ego->ivs, ego->ovs);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(stride_destroy)(ego->is);
+     X(stride_destroy)(ego->os);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     const S *s = ego->slv;
+     const kdft_desc *d = s->desc;
+
+     p->print(p, "(dft-direct-%d%v \"%s\")", d->sz, ego->vl, d->nam);
+}
+
+static int applicable(const solver *ego_, const problem *p_,
+		      const planner *plnr)
+{
+     if (DFTP(p_)) {
+          const S *ego = (const S *) ego_;
+          const problem_dft *p = (const problem_dft *) p_;
+          const kdft_desc *d = ego->desc;
+	  int vl;
+	  int ivs, ovs;
+
+          return (
+	       1
+	       && p->sz->rnk == 1
+	       && p->vecsz->rnk <= 1
+	       && p->sz->dims[0].n == d->sz
+
+	       /* check strides etc */
+	       && X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs)
+
+	       && (d->genus->okp(d, p->ri, p->ii, p->ro, p->io,
+				 p->sz->dims[0].is, p->sz->dims[0].os,
+				 vl, ivs, ovs, plnr))
+
+	       && (0
+		   /* can operate out-of-place */
+		   || p->ri != p->ro
+
+		   /*
+		    * can compute one transform in-place, no matter
+		    * what the strides are.
+		    */
+		   || p->vecsz->rnk == 0
+
+		   /* can operate in-place as long as strides are the same */
+		   || (X(tensor_inplace_strides2)(p->sz, p->vecsz))
+		    )
+	       );
+     }
+
+     return 0;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     P *pln;
+     const problem_dft *p;
+     iodim *d;
+     const kdft_desc *e = ego->desc;
+
+     static const plan_adt padt = {
+	  X(dft_solve), X(null_awake), print, destroy
+     };
+
+     UNUSED(plnr);
+
+     if (!applicable(ego_, p_, plnr))
+          return (plan *)0;
+
+     p = (const problem_dft *) p_;
+
+     pln = MKPLAN_DFT(P, &padt, apply);
+
+     d = p->sz->dims;
+
+     pln->k = ego->k;
+     pln->is = X(mkstride)(e->sz, d[0].is);
+     pln->os = X(mkstride)(e->sz, d[0].os);
+
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     pln->slv = ego;
+
+     X(ops_zero)(&pln->super.super.ops);
+     X(ops_madd2)(pln->vl / e->genus->vl, &e->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+solver *X(mksolver_dft_direct)(kdft k, const kdft_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->k = k;
+     slv->desc = desc;
+     return &(slv->super);
+}
diff --git a/src/fftw3/dft/generic.c b/src/fftw3/dft/generic.c
new file mode 100644
index 0000000..eec32bd
--- /dev/null
+++ b/src/fftw3/dft/generic.c
@@ -0,0 +1,204 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "dft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_dft super;
+     plan *cld;
+     twid *td;
+     int os;
+     int r, m;
+} P;
+
+/***************************************************************************/
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int n, m, r, j;
+     int os, osm;
+     E *buf;
+     const R *W;
+
+     {
+	  plan_dft *cld = (plan_dft *) ego->cld;
+	  cld->apply((plan *) cld, ri, ii, ro, io);
+     }
+
+     r = ego->r;
+
+     STACK_MALLOC(E *, buf, r * 2 * sizeof(E));
+     
+     osm = (m = ego->m) * (os = ego->os);
+     n = m * r;
+     W = ego->td->W;
+     for (j = 0; j < m; ++j, ro += os, io += os) {
+	  int k;
+	  for (k = 0; k < r; ++k) {
+	       E rb = ro[0], ib = io[0];
+	       int i, iw, iw_inc = j + m * k;
+	       for (i = 1, iw = iw_inc; i < r; ++i) {
+		    E xr = ro[i*osm], xi = io[i*osm];
+		    E wr = W[2*iw], wi = W[2*iw+1];
+		    /* note that W[iw] is the product of the DIT twiddle
+		       factor and the size-r DFT twiddle factor */
+		    rb += xr * wr - xi * wi;
+		    ib += xr * wi + xi * wr;
+		    iw += iw_inc;
+		    if (iw >= n)
+			 iw -= n;
+	       }
+	       buf[2*k] = rb;
+	       buf[2*k+1] = ib;
+	  }
+	  for (k = 0; k < r; ++k) {
+	       ro[k*osm] = buf[2*k];
+	       io[k*osm] = buf[2*k+1];
+	  }
+     }
+
+     STACK_FREE(buf);
+}
+
+/***************************************************************************/
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     static const tw_instr generic_tw[] = {
+	  { TW_GENERIC, 0, 0 },
+	  { TW_NEXT, 1, 0 }
+     };
+
+     AWAKE(ego->cld, flg);
+     X(twiddle_awake)(flg, &ego->td, generic_tw,
+		      ego->r * ego->m, ego->r, ego->m);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+
+     p->print(p, "(dft-generic-dit-%d%(%p%))", ego->r, ego->cld);
+}
+
+static int applicable0(const problem *p_)
+{
+     if (DFTP(p_)) {
+          const problem_dft *p = (const problem_dft *) p_;
+          return (1
+	       && p->sz->rnk == 1
+	       && p->vecsz->rnk == 0
+	       && p->sz->dims[0].n > 1
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p_, 
+		      const planner *plnr)
+{
+     UNUSED(ego);
+     if (NO_UGLYP(plnr)) return 0; /* always ugly */
+     if (!applicable0(p_)) return 0;
+
+     if (NO_LARGE_GENERICP(plnr)) {
+          const problem_dft *p = (const problem_dft *) p_;
+	  if (X(first_divisor)(p->sz->dims[0].n) >= GENERIC_MIN_BAD) return 0; 
+     }
+     return 1;
+}
+
+static plan *mkplan(const solver *ego, const problem *p_, planner *plnr)
+{
+     const problem_dft *p = (const problem_dft *) p_;
+     P *pln = 0;
+     int n, r, m;
+     int is, os;
+     plan *cld = (plan *) 0;
+
+     static const plan_adt padt = {
+	  X(dft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego, p_, plnr))
+          goto nada;
+
+     n = p->sz->dims[0].n;
+     is = p->sz->dims[0].is;
+     os = p->sz->dims[0].os;
+
+     r = X(first_divisor)(n);
+     m = n / r;
+
+     cld = X(mkplan_d)(plnr, 
+		       X(mkproblem_dft_d)(X(mktensor_1d)(m, r * is, os),
+					  X(mktensor_1d)(r, is, m * os),
+					  p->ri, p->ii, p->ro, p->io));
+     if (!cld) goto nada;
+
+     pln = MKPLAN_DFT(P, &padt, apply);
+
+     pln->os = os;
+     pln->r = r;
+     pln->m = m;
+     pln->cld = cld;
+     pln->td = 0;
+
+     X(ops_zero)(&pln->super.super.ops);
+     pln->super.super.ops.add = 4 * r * (r-1);
+     pln->super.super.ops.mul = 4 * r * (r-1);
+     /* loads + stores, minus loads + stores for all DIT codelets */
+     pln->super.super.ops.other = 4 * r + 4 * r * r - (6*r - 2);
+     X(ops_madd)(m, &pln->super.super.ops, &cld->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+
+ nada:
+     X(plan_destroy_internal)(cld);
+     X(ifree0)(pln);
+     return (plan *) 0;
+}
+
+/* constructors */
+
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(dft_generic_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/dft/indirect.c b/src/fftw3/dft/indirect.c
new file mode 100644
index 0000000..e00eabf
--- /dev/null
+++ b/src/fftw3/dft/indirect.c
@@ -0,0 +1,238 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: indirect.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+
+/* solvers/plans for vectors of small DFT's that cannot be done
+   in-place directly.  Use a rank-0 plan to rearrange the data
+   before or after the transform.  Can also change an out-of-place
+   plan into a copy + in-place (where the in-place transform
+   is e.g. unit stride). */
+
+/* FIXME: merge with rank-geq2.c(?), since this is just a special case
+   of a rank split where the first/second transform has rank 0. */
+
+#include "dft.h"
+
+typedef problem *(*mkcld_t) (const problem_dft *p);
+
+typedef struct {
+     dftapply apply;
+     problem *(*mkcld)(const problem_dft *p);
+     const char *nam;
+} ndrct_adt;
+
+typedef struct {
+     solver super;
+     const ndrct_adt *adt;
+} S;
+
+typedef struct {
+     plan_dft super;
+     plan *cldcpy, *cld;
+     const S *slv;
+} P;
+
+/*-----------------------------------------------------------------------*/
+/* first rearrange, then transform */
+static void apply_before(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+
+     {
+          plan_dft *cldcpy = (plan_dft *) ego->cldcpy;
+          cldcpy->apply(ego->cldcpy, ri, ii, ro, io);
+     }
+     {
+          plan_dft *cld = (plan_dft *) ego->cld;
+          cld->apply(ego->cld, ro, io, ro, io);
+     }
+}
+
+static problem *mkcld_before(const problem_dft *p)
+{
+     return X(mkproblem_dft_d)(X(tensor_copy_inplace)(p->sz, INPLACE_OS),
+			       X(tensor_copy_inplace)(p->vecsz, INPLACE_OS),
+			       p->ro, p->io, p->ro, p->io);
+}
+
+static const ndrct_adt adt_before =
+{
+     apply_before, mkcld_before, "dft-indirect-before"
+};
+
+/*-----------------------------------------------------------------------*/
+/* first transform, then rearrange */
+
+static void apply_after(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+
+     {
+          plan_dft *cld = (plan_dft *) ego->cld;
+          cld->apply(ego->cld, ri, ii, ri, ii);
+     }
+     {
+          plan_dft *cldcpy = (plan_dft *) ego->cldcpy;
+          cldcpy->apply(ego->cldcpy, ri, ii, ro, io);
+     }
+}
+
+static problem *mkcld_after(const problem_dft *p)
+{
+     return X(mkproblem_dft_d)(X(tensor_copy_inplace)(p->sz, INPLACE_IS),
+			       X(tensor_copy_inplace)(p->vecsz, INPLACE_IS),
+			       p->ri, p->ii, p->ri, p->ii);
+}
+
+static const ndrct_adt adt_after =
+{
+     apply_after, mkcld_after, "dft-indirect-after"
+};
+
+/*-----------------------------------------------------------------------*/
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+     X(plan_destroy_internal)(ego->cldcpy);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cldcpy, flg);
+     AWAKE(ego->cld, flg);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     const S *s = ego->slv;
+     p->print(p, "(%s%(%p%)%(%p%))", s->adt->nam, ego->cld, ego->cldcpy);
+}
+
+static int applicable0(const solver *ego_, const problem *p_,
+		       const planner *plnr)
+{
+     if (DFTP(p_)) {
+	  const S *ego = (const S *) ego_;
+          const problem_dft *p = (const problem_dft *) p_;
+          return (1
+                  && FINITE_RNK(p->vecsz->rnk)
+
+                  /* problem must be a nontrivial transform, not just a copy */
+                  && p->sz->rnk > 0
+
+                  && (0
+
+		      /* problem must be in-place & require some
+		         rearrangement of the data */
+		      || (p->ri == p->ro
+			  && !(X(tensor_inplace_strides2)(p->sz, p->vecsz)))
+
+		      /* or problem must be out of place, transforming
+			 from stride 1/2 to bigger stride, for apply_after */
+		      || (p->ri != p->ro && ego->adt->apply == apply_after
+			  && DESTROY_INPUTP(plnr)
+			  && X(tensor_min_istride)(p->sz) <= 2
+			  && X(tensor_min_ostride)(p->sz) > 2)
+			  
+		      /* or problem must be out of place, transforming
+			 to stride 1/2 from bigger stride, for apply_before */
+		      || (p->ri != p->ro && ego->adt->apply == apply_before
+			  && X(tensor_min_ostride)(p->sz) <= 2
+			  && X(tensor_min_istride)(p->sz) > 2)
+		       )
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego_, const problem *p_,
+		      const planner *plnr)
+{
+     if (!applicable0(ego_, p_, plnr)) return 0;
+     {
+          const problem_dft *p = (const problem_dft *) p_;
+	  if (NO_INDIRECT_OP_P(plnr) && p->ri != p->ro) return 0;
+     }
+     return 1;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const problem_dft *p = (const problem_dft *) p_;
+     const S *ego = (const S *) ego_;
+     P *pln;
+     plan *cld = 0, *cldcpy = 0;
+
+     static const plan_adt padt = {
+	  X(dft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+          return (plan *) 0;
+
+     plnr->planner_flags |= NO_BUFFERING;
+
+     cldcpy = X(mkplan_d)(plnr, 
+			  X(mkproblem_dft_d)(X(mktensor_0d)(),
+					     X(tensor_append)(p->vecsz, p->sz),
+					     p->ri, p->ii, p->ro, p->io));
+     if (!cldcpy) goto nada;
+
+     cld = X(mkplan_d)(plnr, ego->adt->mkcld(p));
+     if (!cld) goto nada;
+
+     pln = MKPLAN_DFT(P, &padt, ego->adt->apply);
+     pln->cld = cld;
+     pln->cldcpy = cldcpy;
+     pln->slv = ego;
+     X(ops_add)(&cld->ops, &cldcpy->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+
+ nada:
+     X(plan_destroy_internal)(cld);
+     X(plan_destroy_internal)(cldcpy);
+     return (plan *)0;
+}
+
+static solver *mksolver(const ndrct_adt *adt)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->adt = adt;
+     return &(slv->super);
+}
+
+void X(dft_indirect_register)(planner *p)
+{
+     unsigned i;
+     static const ndrct_adt *const adts[] = {
+	  &adt_before, &adt_after
+     };
+
+     for (i = 0; i < sizeof(adts) / sizeof(adts[0]); ++i)
+          REGISTER_SOLVER(p, mksolver(adts[i]));
+}
diff --git a/src/fftw3/dft/kdft-dif.c b/src/fftw3/dft/kdft-dif.c
new file mode 100644
index 0000000..c6fad12
--- /dev/null
+++ b/src/fftw3/dft/kdft-dif.c
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: kdft-dif.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+#include "dft.h"
+
+void (*X(kdft_dif_register_hook))(planner *, kdft_dit, const ct_desc *) = 0;
+
+void X(kdft_dif_register)(planner *p, kdft_dif codelet, const ct_desc *desc)
+{
+     REGISTER_SOLVER(p, X(mksolver_dft_ct_dif)(codelet, desc));
+     if (X(kdft_dif_register_hook))
+	  X(kdft_dif_register_hook)(p, codelet, desc);
+}
diff --git a/src/fftw3/dft/kdft-difsq.c b/src/fftw3/dft/kdft-difsq.c
new file mode 100644
index 0000000..aa930ba
--- /dev/null
+++ b/src/fftw3/dft/kdft-difsq.c
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: kdft-difsq.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+#include "dft.h"
+
+void X(kdft_difsq_register)(planner *p, kdft_difsq k, const ct_desc *desc)
+{
+     REGISTER_SOLVER(p, X(mksolver_dft_ct_ditf)(k, desc));
+}
diff --git a/src/fftw3/dft/kdft-dit.c b/src/fftw3/dft/kdft-dit.c
new file mode 100644
index 0000000..054496c
--- /dev/null
+++ b/src/fftw3/dft/kdft-dit.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: kdft-dit.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+#include "dft.h"
+
+void (*X(kdft_dit_register_hook))(planner *, kdft_dit, const ct_desc *) = 0;
+
+void X(kdft_dit_register)(planner *p, kdft_dit codelet, const ct_desc *desc)
+{
+     REGISTER_SOLVER(p, X(mksolver_dft_ct_dit)(codelet, desc));
+     REGISTER_SOLVER(p, X(mksolver_dft_ct_ditbuf)(codelet, desc));
+     if (X(kdft_dit_register_hook))
+	  X(kdft_dit_register_hook)(p, codelet, desc);
+}
diff --git a/src/fftw3/dft/kdft.c b/src/fftw3/dft/kdft.c
new file mode 100644
index 0000000..1f0f57f
--- /dev/null
+++ b/src/fftw3/dft/kdft.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: kdft.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+#include "dft.h"
+
+void X(kdft_register)(planner *p, kdft codelet, const kdft_desc *desc)
+{
+     solver *s = X(mksolver_dft_direct)(codelet, desc);
+     REGISTER_SOLVER(p, s);
+}
diff --git a/src/fftw3/dft/nop.c b/src/fftw3/dft/nop.c
new file mode 100644
index 0000000..9ff8161
--- /dev/null
+++ b/src/fftw3/dft/nop.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: nop.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* plans for vrank -infty DFTs (nothing to do) */
+
+#include "dft.h"
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     UNUSED(ego_);
+     UNUSED(ri);
+     UNUSED(ii);
+     UNUSED(ro);
+     UNUSED(io);
+}
+
+static int applicable(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (DFTP(p_)) {
+          const problem_dft *p = (const problem_dft *) p_;
+          return 0
+	       /* case 1 : -infty vector rank */
+	       || (!FINITE_RNK(p->vecsz->rnk))
+
+	       /* case 2 : rank-0 in-place dft */
+	       || (1
+		   && p->sz->rnk == 0
+		   && FINITE_RNK(p->vecsz->rnk)
+		   && p->ro == p->ri
+		   && X(tensor_inplace_strides)(p->vecsz)
+                    );
+     }
+     return 0;
+}
+
+static void print(const plan *ego, printer *p)
+{
+     UNUSED(ego);
+     p->print(p, "(dft-nop)");
+}
+
+static plan *mkplan(const solver *ego, const problem *p, planner *plnr)
+{
+     static const plan_adt padt = {
+	  X(dft_solve), X(null_awake), print, X(plan_null_destroy)
+     };
+     plan_dft *pln;
+
+     UNUSED(plnr);
+
+     if (!applicable(ego, p))
+          return (plan *) 0;
+     pln = MKPLAN_DFT(plan_dft, &padt, apply);
+     X(ops_zero)(&pln->super.ops);
+
+     return &(pln->super);
+}
+
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     return MKSOLVER(solver, &sadt);
+}
+
+void X(dft_nop_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/dft/plan.c b/src/fftw3/dft/plan.c
new file mode 100644
index 0000000..090a5fb
--- /dev/null
+++ b/src/fftw3/dft/plan.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: plan.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+#include "dft.h"
+
+plan *X(mkplan_dft)(size_t size, const plan_adt *adt, dftapply apply)
+{
+     plan_dft *ego;
+
+     ego = (plan_dft *) X(mkplan)(size, adt);
+     ego->apply = apply;
+
+     return &(ego->super);
+}
diff --git a/src/fftw3/dft/problem.c b/src/fftw3/dft/problem.c
new file mode 100644
index 0000000..51822ad
--- /dev/null
+++ b/src/fftw3/dft/problem.c
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: problem.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+#include "dft.h"
+#include <stddef.h>
+
+static void destroy(problem *ego_)
+{
+     problem_dft *ego = (problem_dft *) ego_;
+     X(tensor_destroy2)(ego->vecsz, ego->sz);
+     X(ifree)(ego_);
+}
+
+static void hash(const problem *p_, md5 *m)
+{
+     const problem_dft *p = (const problem_dft *) p_;
+     X(md5puts)(m, "dft");
+     X(md5int)(m, p->ri == p->ro);
+     X(md5ptrdiff)(m, p->ii - p->ri);
+     X(md5ptrdiff)(m, p->io - p->ro);
+     X(md5int)(m, X(alignment_of)(p->ri));
+     X(md5int)(m, X(alignment_of)(p->ii));
+     X(md5int)(m, X(alignment_of)(p->ro));
+     X(md5int)(m, X(alignment_of)(p->io));
+     X(tensor_md5)(m, p->sz);
+     X(tensor_md5)(m, p->vecsz);
+}
+
+static void print(problem *ego_, printer *p)
+{
+     const problem_dft *ego = (const problem_dft *) ego_;
+     p->print(p, "(dft %d %d %d %td %td %T %T)", 
+	      ego->ri == ego->ro,
+	      X(alignment_of)(ego->ri),
+	      X(alignment_of)(ego->ro),
+	      ego->ii - ego->ri, 
+	      ego->io - ego->ro,
+	      ego->sz,
+	      ego->vecsz);
+}
+
+static void zero(const problem *ego_)
+{
+     const problem_dft *ego = (const problem_dft *) ego_;
+     tensor *sz = X(tensor_append)(ego->vecsz, ego->sz);
+     X(dft_zerotens)(sz, UNTAINT(ego->ri), UNTAINT(ego->ii));
+     X(tensor_destroy)(sz);
+}
+
+static const problem_adt padt =
+{
+     hash,
+     zero,
+     print,
+     destroy
+};
+
+int X(problem_dft_p)(const problem *p)
+{
+     return (p->adt == &padt);
+}
+
+problem *X(mkproblem_dft)(const tensor *sz, const tensor *vecsz,
+                          R *ri, R *ii, R *ro, R *io)
+{
+     problem_dft *ego =
+          (problem_dft *)X(mkproblem)(sizeof(problem_dft), &padt);
+
+     A((ri == ro) == (ii == io)); /* both in place or both out of place */
+     A(X(tensor_kosherp)(sz));
+     A(X(tensor_kosherp)(vecsz));
+
+     /* enforce pointer equality if untainted pointers are equal */
+     if (UNTAINT(ri) == UNTAINT(ro))
+	  ri = ro = JOIN_TAINT(ri, ro);
+     if (UNTAINT(ii) == UNTAINT(io))
+	  ii = io = JOIN_TAINT(ii, io);
+
+     /* more correctness conditions: */
+     A(TAINTOF(ri) == TAINTOF(ii));
+     A(TAINTOF(ro) == TAINTOF(io));
+
+     ego->sz = X(tensor_compress)(sz);
+     ego->vecsz = X(tensor_compress_contiguous)(vecsz);
+     ego->ri = ri;
+     ego->ii = ii;
+     ego->ro = ro;
+     ego->io = io;
+
+     A(FINITE_RNK(ego->sz->rnk));
+     return &(ego->super);
+}
+
+/* Same as X(mkproblem_dft), but also destroy input tensors. */
+problem *X(mkproblem_dft_d)(tensor *sz, tensor *vecsz,
+                            R *ri, R *ii, R *ro, R *io)
+{
+     problem *p;
+     p = X(mkproblem_dft)(sz, vecsz, ri, ii, ro, io);
+     X(tensor_destroy2)(vecsz, sz);
+     return p;
+}
diff --git a/src/fftw3/dft/rader-omega.c b/src/fftw3/dft/rader-omega.c
new file mode 100644
index 0000000..7fd86b9
--- /dev/null
+++ b/src/fftw3/dft/rader-omega.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "dft.h"
+
+static rader_tl *omegas = 0;
+
+R *X(dft_rader_mkomega)(plan *p_, int n, int ginv)
+{
+     plan_dft *p = (plan_dft *) p_;
+     R *omega;
+     int i, gpower;
+     trigreal scale;
+
+     if ((omega = X(rader_tl_find)(n, n, ginv, omegas)))
+	  return omega;
+
+     omega = (R *)MALLOC(sizeof(R) * (n - 1) * 2, TWIDDLES);
+
+     scale = n - 1.0; /* normalization for convolution */
+
+     for (i = 0, gpower = 1; i < n-1; ++i, gpower = MULMOD(gpower, ginv, n)) {
+	  omega[2*i] = X(cos2pi)(gpower, n) / scale;
+	  omega[2*i+1] = FFT_SIGN * X(sin2pi)(gpower, n) / scale;
+     }
+     A(gpower == 1);
+
+     AWAKE(p_, 1);
+     p->apply(p_, omega, omega + 1, omega, omega + 1);
+     AWAKE(p_, 0);
+
+     X(rader_tl_insert)(n, n, ginv, omega, &omegas);
+     return omega;
+}
+
+void X(dft_rader_free_omega)(R **omega)
+{
+     X(rader_tl_delete)(*omega, &omegas);
+     *omega = 0;
+}
diff --git a/src/fftw3/dft/rader.c b/src/fftw3/dft/rader.c
new file mode 100644
index 0000000..f31f370
--- /dev/null
+++ b/src/fftw3/dft/rader.c
@@ -0,0 +1,491 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "dft.h"
+
+/*
+ * Compute transforms of prime sizes using Rader's trick: turn them
+ * into convolutions of size n - 1, which you then perform via a pair
+ * of FFTs.   This file contains both nontwiddle (direct) and 
+ * twiddle (DIT Cooley-Tukey) solvers.
+ */
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_dft super;
+
+     plan *cld1, *cld2;
+     R *omega;
+     int n, g, ginv;
+     int is, os;
+     plan *cld_omega;
+} P;
+
+typedef struct {
+     P super;
+     plan *cld;
+     R *W;
+     int os;
+     int m;
+} P_dit;
+
+
+static rader_tl *twiddles = 0;
+
+/***************************************************************************/
+
+/* Below, we extensively use the identity that fft(x*)* = ifft(x) in
+   order to share data between forward and backward transforms and to
+   obviate the necessity of having separate forward and backward
+   plans.  (Although we often compute separate plans these days anyway
+   due to the differing strides, etcetera.)
+
+   Of course, since the new FFTW gives us separate pointers to
+   the real and imaginary parts, we could have instead used the
+   fft(r,i) = ifft(i,r) form of this identity, but it was easier to
+   reuse the code from our old version. */
+
+static void apply_aux(int r, int ginv, plan *cld1,plan *cld2, const R *omega,
+		      R *buf, R r0, R i0, R *ro, R *io, int os)
+{
+     int gpower, k;
+
+     /* compute DFT of buf, storing in output (except DC): */
+     {
+	    plan_dft *cld = (plan_dft *) cld1;
+	    cld->apply(cld1, buf, buf+1, ro+os, io+os);
+     }
+
+     /* set output DC component: */
+     ro[0] = r0 + ro[os];
+     io[0] = i0 + io[os];
+
+     /* now, multiply by omega: */
+     for (k = 0; k < r - 1; ++k) {
+	  E rB, iB, rW, iW;
+	  rW = omega[2*k];
+	  iW = omega[2*k+1];
+	  rB = ro[(k+1)*os];
+	  iB = io[(k+1)*os];
+	  ro[(k+1)*os] = rW * rB - iW * iB;
+	  io[(k+1)*os] = -(rW * iB + iW * rB);
+     }
+     
+     /* this will add input[0] to all of the outputs after the ifft */
+     ro[os] += r0;
+     io[os] -= i0;
+
+     /* inverse FFT: */
+     {
+	    plan_dft *cld = (plan_dft *) cld2;
+	    cld->apply(cld2, ro+os, io+os, buf, buf+1);
+     }
+     
+     /* finally, do inverse permutation to unshuffle the output: */
+     gpower = 1;
+     for (k = 0; k < r - 1; ++k, gpower = MULMOD(gpower, ginv, r)) {
+	  ro[gpower * os] = buf[2*k];
+	  io[gpower * os] = -buf[2*k+1];
+     }
+     A(gpower == 1);
+}
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int is;
+     int k, gpower, g, r;
+     R *buf;
+
+     r = ego->n; is = ego->is; g = ego->g; 
+     buf = (R *) MALLOC(sizeof(R) * (r - 1) * 2, BUFFERS);
+
+     /* First, permute the input, storing in buf: */
+     for (gpower = 1, k = 0; k < r - 1; ++k, gpower = MULMOD(gpower, g, r)) {
+	  R rA, iA;
+	  rA = ri[gpower * is];
+	  iA = ii[gpower * is];
+	  buf[2*k] = rA; buf[2*k + 1] = iA;
+     }
+     /* gpower == g^(r-1) mod r == 1 */;
+
+     apply_aux(r, ego->ginv, ego->cld1, ego->cld2, ego->omega, 
+	       buf, ri[0], ii[0], ro, io, ego->os);
+
+     X(ifree)(buf);
+}
+
+static void apply_dit(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P_dit *ego_dit = (const P_dit *) ego_;
+     const P *ego;
+     plan *cld1, *cld2;
+     int os, osm;
+     int j, k, gpower, g, ginv, r, m;
+     R *buf;
+     const R *omega, *W;
+
+     {
+	   plan *cld0 = ego_dit->cld;
+	   plan_dft *cld = (plan_dft *) cld0;
+	   cld->apply(cld0, ri, ii, ro, io);
+     }
+
+     ego = (const P *) ego_;
+     cld1 = ego->cld1;
+     cld2 = ego->cld2;
+     r = ego->n;
+     m = ego_dit->m;
+     g = ego->g; 
+     ginv = ego->ginv;
+     omega = ego->omega;
+     W = ego_dit->W;
+     os = ego_dit->os;
+     osm = ego->os;
+     gpower = 1;
+
+     buf = (R *) MALLOC(sizeof(R) * (r - 1) * 2, BUFFERS);
+
+     for (j = 0; j < m; ++j, ro += os, io += os, W += 2*(r - 1)) {
+	  /* First, permute the input and multiply by W, storing in buf: */
+	  A(gpower == 1);
+	  for (k = 0; k < r - 1; ++k, gpower = MULMOD(gpower, g, r)) {
+	       E rA, iA, rW, iW;
+	       rA = ro[gpower * osm];
+	       iA = io[gpower * osm];
+	       rW = W[2*k];
+	       iW = W[2*k+1];
+	       buf[2*k] = rW * rA - iW * iA;
+	       buf[2*k + 1] = rW * iA + iW * rA;
+	  }
+	  /* gpower == g^(r-1) mod r == 1 */;
+	  
+	  apply_aux(r, ginv, cld1, cld2, omega, 
+		    buf, ro[0], io[0], ro, io, osm);
+     }
+
+     X(ifree)(buf);
+}
+
+static R *mktwiddle(int m, int r, int g)
+{
+     int i, j, gpower;
+     int n = r * m;
+     R *W;
+
+     if ((W = X(rader_tl_find)(m, r, g, twiddles)))
+	  return W;
+
+     W = (R *)MALLOC(sizeof(R) * (r - 1) * m * 2, TWIDDLES);
+     for (i = 0; i < m; ++i) {
+	  for (gpower = 1, j = 0; j < r - 1;
+	       ++j, gpower = MULMOD(gpower, g, r)) {
+	       int k = i * (r - 1) + j;
+	       W[2*k] = X(cos2pi)(i * gpower, n);
+	       W[2*k+1] = FFT_SIGN * X(sin2pi)(i * gpower, n);
+	  }
+	  A(gpower == 1);
+     }
+
+     X(rader_tl_insert)(m, r, g, W, &twiddles);
+     return W;
+}
+
+static void free_twiddle(R *twiddle)
+{
+     X(rader_tl_delete)(twiddle, &twiddles);
+}
+
+/***************************************************************************/
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+
+     AWAKE(ego->cld1, flg);
+     AWAKE(ego->cld2, flg);
+
+     if (flg) {
+	  if (!ego->omega) 
+	       ego->omega = 
+		    X(dft_rader_mkomega)(ego->cld_omega, ego->n, ego->ginv);
+     } else {
+	  X(dft_rader_free_omega)(&ego->omega);
+     }
+}
+
+static void awake_dit(plan *ego_, int flg)
+{
+     P_dit *ego = (P_dit *) ego_;
+
+     AWAKE(ego->cld, flg);
+     if (flg)
+	  ego->W = mktwiddle(ego->m, ego->super.n, ego->super.g);
+     else {
+	  free_twiddle(ego->W);
+	  ego->W = 0;
+     }
+
+     awake(ego_, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld_omega);
+     X(plan_destroy_internal)(ego->cld2);
+     X(plan_destroy_internal)(ego->cld1);
+}
+
+static void destroy_dit(plan *ego_)
+{
+     P_dit *ego = (P_dit *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+     destroy(ego_);
+}
+
+static void print_aux(const char *name, const P *ego, printer *p)
+{
+     p->print(p, "(%s-%d%ois=%oos=%(%p%)",
+              name, ego->n, ego->is, ego->os, ego->cld1);
+     if (ego->cld2 != ego->cld1)
+          p->print(p, "%(%p%)", ego->cld2);
+     if (ego->cld_omega != ego->cld1 && ego->cld_omega != ego->cld2)
+          p->print(p, "%(%p%)", ego->cld_omega);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     print_aux("dft-rader", (const P *) ego_, p);
+     p->putchr(p, ')');
+}
+
+static void print_dit(const plan *ego_, printer *p)
+{
+     const P_dit *ego_dit = (const P_dit *) ego_;
+
+     print_aux("dft-rader-dit", (const P *) ego_, p);
+     p->print(p, "%(%p%))", ego_dit->cld);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (DFTP(p_)) {
+          const problem_dft *p = (const problem_dft *) p_;
+          return (1
+	       && p->sz->rnk == 1
+	       && p->vecsz->rnk == 0
+	       && X(is_prime)(p->sz->dims[0].n)
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable0_dit(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (DFTP(p_)) {
+          const problem_dft *p = (const problem_dft *) p_;
+          return (1
+	       && p->sz->rnk == 1
+	       && p->vecsz->rnk == 0
+	       && p->sz->dims[0].n > 1
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego_, const problem *p_,
+		      const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0(ego_, p_));
+}
+
+static int applicable_dit(const solver *ego_, const problem *p_, 
+			  const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0_dit(ego_, p_));
+}
+
+static int mkP(P *pln, int n, int is, int os, R *ro, R *io,
+	       planner *plnr)
+{
+     plan *cld1 = (plan *) 0;
+     plan *cld2 = (plan *) 0;
+     plan *cld_omega = (plan *) 0;
+     R *buf = (R *) 0;
+
+     /* initial allocation for the purpose of planning */
+     buf = (R *) MALLOC(sizeof(R) * (n - 1) * 2, BUFFERS);
+
+     cld1 = X(mkplan_d)(plnr, 
+			X(mkproblem_dft_d)(X(mktensor_1d)(n - 1, 2, os),
+					   X(mktensor_1d)(1, 0, 0),
+  					   buf, buf + 1, ro + os, io + os));
+     if (!cld1) goto nada;
+
+     cld2 = X(mkplan_d)(plnr, 
+			X(mkproblem_dft_d)(X(mktensor_1d)(n - 1, os, 2),
+					   X(mktensor_1d)(1, 0, 0),
+  					   ro + os, io + os, buf, buf + 1));
+
+     if (!cld2) goto nada;
+
+     /* plan for omega array */
+     plnr->planner_flags |= ESTIMATE;
+     cld_omega = X(mkplan_d)(plnr, 
+			     X(mkproblem_dft_d)(X(mktensor_1d)(n - 1, 2, 2),
+						X(mktensor_1d)(1, 0, 0),
+						buf, buf + 1, buf, buf + 1));
+     if (!cld_omega) goto nada;
+
+     /* deallocate buffers; let awake() or apply() allocate them for real */
+     X(ifree)(buf);
+     buf = 0;
+
+     pln->cld1 = cld1;
+     pln->cld2 = cld2;
+     pln->cld_omega = cld_omega;
+     pln->omega = 0;
+     pln->n = n;
+     pln->is = is;
+     pln->os = os;
+     pln->g = X(find_generator)(n);
+     pln->ginv = X(power_mod)(pln->g, n - 2, n);
+     A(MULMOD(pln->g, pln->ginv, n) == 1);
+
+     X(ops_add)(&cld1->ops, &cld2->ops, &pln->super.super.ops);
+     pln->super.super.ops.other += (n - 1) * (4 * 2 + 6) + 6;
+     pln->super.super.ops.add += (n - 1) * 2 + 4;
+     pln->super.super.ops.mul += (n - 1) * 4;
+
+     return 1;
+
+ nada:
+     X(ifree0)(buf);
+     X(plan_destroy_internal)(cld_omega);
+     X(plan_destroy_internal)(cld2);
+     X(plan_destroy_internal)(cld1);
+     return 0;
+}
+
+static plan *mkplan(const solver *ego, const problem *p_, planner *plnr)
+{
+     const problem_dft *p = (const problem_dft *) p_;
+     P *pln;
+     int n;
+     int is, os;
+
+     static const plan_adt padt = {
+	  X(dft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego, p_, plnr))
+	  return (plan *) 0;
+
+     n = p->sz->dims[0].n;
+     is = p->sz->dims[0].is;
+     os = p->sz->dims[0].os;
+
+     pln = MKPLAN_DFT(P, &padt, apply);
+     if (!mkP(pln, n, is, os, p->ro, p->io, plnr)) {
+	  X(ifree)(pln);
+	  return (plan *) 0;
+     }
+     return &(pln->super.super);
+}
+
+static plan *mkplan_dit(const solver *ego, const problem *p_, planner *plnr)
+{
+     const problem_dft *p = (const problem_dft *) p_;
+     P_dit *pln = 0;
+     int n, r, m;
+     int is, os;
+     plan *cld = (plan *) 0;
+
+     static const plan_adt padt = {
+	  X(dft_solve), awake_dit, print_dit, destroy_dit
+     };
+
+     if (!applicable_dit(ego, p_, plnr))
+          goto nada;
+
+     n = p->sz->dims[0].n;
+     is = p->sz->dims[0].is;
+     os = p->sz->dims[0].os;
+
+     r = X(first_divisor)(n);
+     m = n / r;
+
+     cld = X(mkplan_d)(plnr, 
+		       X(mkproblem_dft_d)(X(mktensor_1d)(m, r * is, os),
+					  X(mktensor_1d)(r, is, m * os),
+					  p->ri, p->ii, p->ro, p->io));
+     if (!cld) goto nada;
+
+     pln = MKPLAN_DFT(P_dit, &padt, apply_dit);
+     if (!mkP(&pln->super, r, os*m, os*m, p->ro, p->io, plnr))
+	  goto nada;
+
+     pln->os = os;
+     pln->m = m;
+     pln->cld = cld;
+     pln->W = 0;
+
+     pln->super.super.super.ops.add += 2 * (r-1);
+     pln->super.super.super.ops.mul += 4 * (r-1);
+     X(ops_madd)(m, &pln->super.super.super.ops, &cld->ops,
+		 &pln->super.super.super.ops);
+
+     return &(pln->super.super.super);
+
+ nada:
+     X(plan_destroy_internal)(cld);
+     X(ifree0)(pln);
+     return (plan *) 0;
+}
+
+/* constructors */
+
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+static solver *mksolver_dit(void)
+{
+     static const solver_adt sadt = { mkplan_dit };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(dft_rader_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+     REGISTER_SOLVER(p, mksolver_dit());
+}
diff --git a/src/fftw3/dft/rank-geq2.c b/src/fftw3/dft/rank-geq2.c
new file mode 100644
index 0000000..81dffd9
--- /dev/null
+++ b/src/fftw3/dft/rank-geq2.c
@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rank-geq2.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* plans for DFT of rank >= 2 (multidimensional) */
+
+#include "dft.h"
+
+typedef struct {
+     solver super;
+     int spltrnk;
+     const int *buddies;
+     int nbuddies;
+} S;
+
+typedef struct {
+     plan_dft super;
+
+     plan *cld1, *cld2;
+     const S *solver;
+} P;
+
+/* Compute multi-dimensional DFT by applying the two cld plans
+   (lower-rnk DFTs). */
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     plan_dft *cld1, *cld2;
+
+     cld1 = (plan_dft *) ego->cld1;
+     cld1->apply(ego->cld1, ri, ii, ro, io);
+
+     cld2 = (plan_dft *) ego->cld2;
+     cld2->apply(ego->cld2, ro, io, ro, io);
+}
+
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cld1, flg);
+     AWAKE(ego->cld2, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld2);
+     X(plan_destroy_internal)(ego->cld1);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     const S *s = ego->solver;
+     p->print(p, "(dft-rank>=2/%d%(%p%)%(%p%))",
+	      s->spltrnk, ego->cld1, ego->cld2);
+}
+
+static int picksplit(const S *ego, const tensor *sz, int *rp)
+{
+     A(sz->rnk > 1); /* cannot split rnk <= 1 */
+     if (!X(pickdim)(ego->spltrnk, ego->buddies, ego->nbuddies, sz, 1, rp))
+	  return 0;
+     *rp += 1; /* convert from dim. index to rank */
+     if (*rp >= sz->rnk) /* split must reduce rank */
+	  return 0;
+     return 1;
+}
+
+static int applicable0(const solver *ego_, const problem *p_, int *rp)
+{
+     if (DFTP(p_)) {
+          const problem_dft *p = (const problem_dft *) p_;
+          const S *ego = (const S *)ego_;
+          return (1
+                  && p->sz->rnk >= 2
+                  && picksplit(ego, p->sz, rp)
+	       );
+     }
+
+     return 0;
+}
+
+/* TODO: revise this. */
+static int applicable(const solver *ego_, const problem *p_, 
+		      const planner *plnr, int *rp)
+{
+     const S *ego = (const S *)ego_;
+     const problem_dft *p = (const problem_dft *) p_;
+
+     if (!applicable0(ego_, p_, rp)) return 0;
+
+     /* fixed spltrnk (unlike fftw2's spltrnk=1, default buddies[0] is
+        spltrnk=0, which is an asymptotic "theoretical optimum" for
+        an ideal cache; it's equivalent to spltrnk=1 for rnk < 4). */
+     if (NO_RANK_SPLITSP(plnr) && (ego->spltrnk != ego->buddies[0])) return 0;
+
+     /* Heuristic: if the vector stride is greater than the transform
+        sz, don't use (prefer to do the vector loop first with a
+        vrank-geq1 plan). */
+     if (NO_UGLYP(plnr))
+	  if (p->vecsz->rnk > 0 &&
+	      X(tensor_min_stride)(p->vecsz) > X(tensor_max_index)(p->sz))
+	       return 0;
+
+     return 1;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     const problem_dft *p;
+     P *pln;
+     plan *cld1 = 0, *cld2 = 0;
+     tensor *sz1, *sz2, *vecszi, *sz2i;
+     int spltrnk;
+
+     static const plan_adt padt = {
+	  X(dft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr, &spltrnk))
+          return (plan *) 0;
+
+     p = (const problem_dft *) p_;
+     X(tensor_split)(p->sz, &sz1, spltrnk, &sz2);
+     vecszi = X(tensor_copy_inplace)(p->vecsz, INPLACE_OS);
+     sz2i = X(tensor_copy_inplace)(sz2, INPLACE_OS);
+
+     cld1 = X(mkplan_d)(plnr, 
+			X(mkproblem_dft_d)(X(tensor_copy)(sz2),
+					   X(tensor_append)(p->vecsz, sz1),
+					   p->ri, p->ii, p->ro, p->io));
+     if (!cld1) goto nada;
+
+     cld2 = X(mkplan_d)(plnr, 
+			X(mkproblem_dft_d)(
+			     X(tensor_copy_inplace)(sz1, INPLACE_OS),
+			     X(tensor_append)(vecszi, sz2i),
+			     p->ro, p->io, p->ro, p->io));
+     if (!cld2) goto nada;
+
+     pln = MKPLAN_DFT(P, &padt, apply);
+
+     pln->cld1 = cld1;
+     pln->cld2 = cld2;
+
+     pln->solver = ego;
+     X(ops_add)(&cld1->ops, &cld2->ops, &pln->super.super.ops);
+
+     X(tensor_destroy4)(sz1, sz2, vecszi, sz2i);
+
+     return &(pln->super.super);
+
+ nada:
+     X(plan_destroy_internal)(cld2);
+     X(plan_destroy_internal)(cld1);
+     X(tensor_destroy4)(sz1, sz2, vecszi, sz2i);
+     return (plan *) 0;
+}
+
+static solver *mksolver(int spltrnk, const int *buddies, int nbuddies)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->spltrnk = spltrnk;
+     slv->buddies = buddies;
+     slv->nbuddies = nbuddies;
+     return &(slv->super);
+}
+
+void X(dft_rank_geq2_register)(planner *p)
+{
+     int i;
+     static const int buddies[] = { 0, 1, -2 };
+
+     const int nbuddies = sizeof(buddies) / sizeof(buddies[0]);
+
+     for (i = 0; i < nbuddies; ++i)
+          REGISTER_SOLVER(p, mksolver(buddies[i], buddies, nbuddies));
+
+     /* FIXME:
+
+        Should we try more buddies? 
+
+        Another possible variant is to swap cld1 and cld2 (or rather,
+        to swap their problems; they are not interchangeable because
+        cld2 must be in-place).  In past versions of FFTW, however, I
+        seem to recall that such rearrangements have made little or no
+        difference.
+     */
+}
diff --git a/src/fftw3/dft/rank0.c b/src/fftw3/dft/rank0.c
new file mode 100644
index 0000000..bef719f
--- /dev/null
+++ b/src/fftw3/dft/rank0.c
@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rank0.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* plans for rank-0 DFTs (copy operations) */
+
+#include "dft.h"
+
+#ifdef HAVE_STRING_H
+#include <string.h>		/* for memcpy() */
+#endif
+
+
+typedef struct {
+     dftapply apply;
+     int (*applicable)(const problem_dft *p);
+     const char *nam;
+} rnk0adt;
+
+typedef struct {
+     solver super;
+     const rnk0adt *adt;
+} S;
+
+typedef struct {
+     plan_dft super;
+     int vl;
+     int ivs, ovs;
+     const S *slv;
+} P;
+
+/* generic applicability function */
+static int applicable(const solver *ego_, const problem *p_)
+{
+     if (DFTP(p_)) {
+          const S *ego = (const S *) ego_;
+          const problem_dft *p = (const problem_dft *) p_;
+          return (1
+		  && p->ri != p->ro
+                  && p->sz->rnk == 0
+                  && ego->adt->applicable(p)
+	       );
+     }
+     return 0;
+}
+
+/*-----------------------------------------------------------------------*/
+/* rank-0 dft, vl == 1: just a copy */
+static void apply_1(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     UNUSED(ego_);
+     *ro = *ri;
+     *io = *ii;
+}
+
+static int applicable_1(const problem_dft *p)
+{
+     return (p->vecsz->rnk == 0);
+}
+
+static const rnk0adt adt_cpy1 =
+{
+     apply_1, applicable_1, "dft-rank0-cpy1"
+};
+
+/*-----------------------------------------------------------------------*/
+/* rank-0 dft, vl > 1: just a copy loop (unroll 4) */
+static void apply_vec(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int i, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     int imi = ii - ri, imo = io - ro;
+     for (i = vl; i > 0; --i) {
+          R r0 = ri[0], i0 = ri[imi];
+          ro[0] = r0; ro[imo] = i0; 
+	  ri += ivs; ro += ovs;
+     }
+}
+
+static int applicable_vec(const problem_dft *p)
+{
+     return (p->vecsz->rnk == 1 && p->ro != p->ri);
+}
+
+static const rnk0adt adt_vec =
+{
+     apply_vec, applicable_vec, "dft-rank0-vec"
+};
+
+/*-----------------------------------------------------------------------*/
+/* rank-0 dft, vl > 1, [io]vs == 1, using memcpy */
+static void apply_io1(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int vl = ego->vl;
+     memcpy(ro, ri, vl * sizeof(R));
+     memcpy(io, ii, vl * sizeof(R));
+}
+
+static int applicable_io1(const problem_dft *p)
+{
+     return (1
+             && applicable_vec(p)
+             && p->vecsz->dims[0].is == 1
+             && p->vecsz->dims[0].os == 1
+	  );
+}
+
+static const rnk0adt adt_io1 =
+{
+     apply_io1, applicable_io1, "dft-rank0-io1-memcpy"
+};
+
+/*-----------------------------------------------------------------------*/
+/* rank-0 dft, vl > 1, [io]vs == 2 (interleaved) using memcpy */
+static void apply_io2r(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int vl = ego->vl;
+     UNUSED(ii);
+     UNUSED(io);		/* i{in,out}put == r{in,out}put + 1 */
+     memcpy(ro, ri, vl * sizeof(R) * 2);
+}
+
+static int applicable_io2r(const problem_dft *p)
+{
+     return (1
+             && applicable_vec(p)
+             && p->vecsz->dims[0].is == 2
+             && p->vecsz->dims[0].os == 2
+             && p->ii == p->ri + 1 && p->io == p->ro + 1
+	  );
+}
+
+static const rnk0adt adt_io2r =
+{
+     apply_io2r, applicable_io2r, "dft-rank0-io2r-memcpy"
+};
+
+static void apply_io2i(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int vl = ego->vl;
+     UNUSED(ri);
+     UNUSED(ro);		/* r{in,out}put == i{in,out}put + 1 */
+     memcpy(io, ii, vl * sizeof(R) * 2);
+}
+
+static int applicable_io2i(const problem_dft *p)
+{
+     return (1
+             && applicable_vec(p)
+             && p->vecsz->dims[0].is == 2
+             && p->vecsz->dims[0].os == 2
+             && p->ri == p->ii + 1 && p->ro == p->io + 1
+	  );
+}
+
+static const rnk0adt adt_io2i =
+{
+     apply_io2i, applicable_io2i, "dft-rank0-io2i-memcpy"
+};
+
+/*-----------------------------------------------------------------------*/
+/* generic stuff: */
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(%s%v)", ego->slv->adt->nam, ego->vl);
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     const problem_dft *p;
+     P *pln;
+
+     static const plan_adt padt = {
+	  X(dft_solve), X(null_awake), print, X(plan_null_destroy)
+     };
+
+     UNUSED(plnr);
+
+     if (!applicable(ego_, p_))
+          return (plan *) 0;
+
+     p = (const problem_dft *) p_;
+     pln = MKPLAN_DFT(P, &padt, ego->adt->apply);
+
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     pln->slv = ego;
+
+     /* 2*vl loads, 2*vl stores */
+     X(ops_other)(4 * pln->vl, &pln->super.super.ops);
+     return &(pln->super.super);
+}
+
+static solver *mksolver(const rnk0adt *adt)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->adt = adt;
+     return &(slv->super);
+}
+
+void X(dft_rank0_register)(planner *p)
+{
+     unsigned i;
+     static const rnk0adt *const adts[] = {
+	  &adt_cpy1, &adt_vec, &adt_io1, &adt_io2r, &adt_io2i
+     };
+
+     for (i = 0; i < sizeof(adts) / sizeof(adts[0]); ++i)
+          REGISTER_SOLVER(p, mksolver(adts[i]));
+}
diff --git a/src/fftw3/dft/solve.c b/src/fftw3/dft/solve.c
new file mode 100644
index 0000000..abcad7f
--- /dev/null
+++ b/src/fftw3/dft/solve.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: solve.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+#include "dft.h"
+
+/* use the apply() operation for DFT problems */
+void X(dft_solve)(const plan *ego_, const problem *p_)
+{
+     const plan_dft *ego = (const plan_dft *) ego_;
+     const problem_dft *p = (const problem_dft *) p_;
+     ego->apply(ego_, 
+		UNTAINT(p->ri), UNTAINT(p->ii), 
+		UNTAINT(p->ro), UNTAINT(p->io));
+}
diff --git a/src/fftw3/dft/vrank-geq1.c b/src/fftw3/dft/vrank-geq1.c
new file mode 100644
index 0000000..92a5776
--- /dev/null
+++ b/src/fftw3/dft/vrank-geq1.c
@@ -0,0 +1,215 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: vrank-geq1.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+
+/* Plans for handling vector transform loops.  These are *just* the
+   loops, and rely on child plans for the actual DFTs.
+ 
+   They form a wrapper around solvers that don't have apply functions
+   for non-null vectors.
+ 
+   vrank-geq1 plans also recursively handle the case of multi-dimensional
+   vectors, obviating the need for most solvers to deal with this.  We
+   can also play games here, such as reordering the vector loops.
+ 
+   Each vrank-geq1 plan reduces the vector rank by 1, picking out a
+   dimension determined by the vecloop_dim field of the solver. */
+
+#include "dft.h"
+
+typedef struct {
+     solver super;
+     int vecloop_dim;
+     const int *buddies;
+     int nbuddies;
+} S;
+
+typedef struct {
+     plan_dft super;
+
+     plan *cld;
+     int vl;
+     int ivs, ovs;
+     const S *solver;
+} P;
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int i, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     dftapply cldapply = ((plan_dft *) ego->cld)->apply;
+
+     for (i = 0; i < vl; ++i) {
+          cldapply(ego->cld,
+                   ri + i * ivs, ii + i * ivs, ro + i * ovs, io + i * ovs);
+     }
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cld, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     const S *s = ego->solver;
+     p->print(p, "(dft-vrank>=1-x%d/%d%(%p%))",
+ 	      ego->vl, s->vecloop_dim, ego->cld);
+}
+
+static int pickdim(const S *ego, const tensor *vecsz, int oop, int *dp)
+{
+     return X(pickdim)(ego->vecloop_dim, ego->buddies, ego->nbuddies,
+		       vecsz, oop, dp);
+}
+
+static int applicable0(const solver *ego_, const problem *p_, int *dp)
+{
+     if (DFTP(p_)) {
+          const S *ego = (const S *) ego_;
+          const problem_dft *p = (const problem_dft *) p_;
+
+          return (1
+                  && FINITE_RNK(p->vecsz->rnk)
+                  && p->vecsz->rnk > 0
+                  && pickdim(ego, p->vecsz, p->ri != p->ro, dp)
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego_, const problem *p_, 
+		      const planner *plnr, int *dp)
+{
+     const S *ego = (const S *)ego_;
+     const problem_dft *p;
+
+     if (!applicable0(ego_, p_, dp)) return 0;
+
+     /* fftw2 behavior */
+     if (NO_VRANK_SPLITSP(plnr) && (ego->vecloop_dim != ego->buddies[0]))
+	  return 0;
+
+     p = (const problem_dft *) p_;
+
+     if (NO_UGLYP(plnr)) {
+	  /* Heuristic: if the transform is multi-dimensional, and the
+	     vector stride is less than the transform size, then we
+	     probably want to use a rank>=2 plan first in order to combine
+	     this vector with the transform-dimension vectors. */
+	  {
+	       iodim *d = p->vecsz->dims + *dp;
+	       if (1
+		   && p->sz->rnk > 1 
+		   && X(imin)(X(iabs)(d->is), X(iabs)(d->os)) 
+		   < X(tensor_max_index)(p->sz)
+		    )
+		    return 0;
+	  }
+
+	  /* Heuristic: don't use a vrank-geq1 for rank-0 vrank-1
+	     transforms, since this case is better handled by rank-0
+	     solvers. */
+	  if (p->sz->rnk == 0 && p->vecsz->rnk == 1) return 0;
+
+	  if (NONTHREADED_ICKYP(plnr)) return 0; /* prefer threaded version */
+     }
+
+     return 1;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     const problem_dft *p;
+     P *pln;
+     plan *cld;
+     int vdim;
+     iodim *d;
+
+     static const plan_adt padt = {
+	  X(dft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr, &vdim))
+          return (plan *) 0;
+     p = (const problem_dft *) p_;
+
+     d = p->vecsz->dims + vdim;
+
+     A(d->n > 1);
+     cld = X(mkplan_d)(plnr,
+		       X(mkproblem_dft_d)(
+			    X(tensor_copy)(p->sz),
+			    X(tensor_copy_except)(p->vecsz, vdim),
+			    TAINT(p->ri, d->is), TAINT(p->ii, d->is),
+			    TAINT(p->ro, d->os), TAINT(p->io, d->os)));
+     if (!cld) return (plan *) 0;
+
+     pln = MKPLAN_DFT(P, &padt, apply);
+
+     pln->cld = cld;
+     pln->vl = d->n;
+     pln->ivs = d->is;
+     pln->ovs = d->os;
+
+     pln->solver = ego;
+     X(ops_zero)(&pln->super.super.ops);
+     pln->super.super.ops.other = 3.14159; /* magic to prefer codelet loops */
+     X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops);
+     pln->super.super.pcost = pln->vl * cld->pcost;
+
+     return &(pln->super.super);
+}
+
+static solver *mksolver(int vecloop_dim, const int *buddies, int nbuddies)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->vecloop_dim = vecloop_dim;
+     slv->buddies = buddies;
+     slv->nbuddies = nbuddies;
+     return &(slv->super);
+}
+
+void X(dft_vrank_geq1_register)(planner *p)
+{
+     int i;
+
+     /* FIXME: Should we try other vecloop_dim values? */
+     static const int buddies[] = { 1, -1 };
+
+     const int nbuddies = sizeof(buddies) / sizeof(buddies[0]);
+
+     for (i = 0; i < nbuddies; ++i)
+          REGISTER_SOLVER(p, mksolver(buddies[i], buddies, nbuddies));
+}
diff --git a/src/fftw3/dft/vrank2-transpose.c b/src/fftw3/dft/vrank2-transpose.c
new file mode 100644
index 0000000..ebed1ff
--- /dev/null
+++ b/src/fftw3/dft/vrank2-transpose.c
@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: vrank2-transpose.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* rank-0, vector-rank-2, square transposition  */
+
+#include "dft.h"
+
+/* transposition routine. TODO: optimize? */
+static void t(R *rA, R *iA, int n, int is, int js)
+{
+     int i, j;
+     int im = iA - rA;
+
+     for (i = 1; i < n; ++i) {
+	  R *p0 = rA + i * is;
+	  R *p1 = rA + i * js;
+          for (j = 0; j < i; ++j) {
+               R ar = p0[0], ai = p0[im];
+               R br = p1[0], bi = p1[im];
+               p1[0] = ar; p1[im] = ai; p1 += is;
+               p0[0] = br; p0[im] = bi; p0 += js;
+          }
+     }
+}
+
+typedef solver S;
+
+typedef struct {
+     plan_dft super;
+     int n;
+     int s0, s1;
+     int m;
+     int offset;
+     int nd, md, d; /* d = gcd(n,m), nd = n / d, md = m / d */
+} P;
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     UNUSED(ro);
+     UNUSED(io);
+     A(ego->n == ego->m);
+     t(ri, ii, ego->n, ego->s0, ego->s1);
+}
+
+static void apply_general(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int nd = ego->nd, md = ego->md, d = ego->d;
+     R *buf = (R *)MALLOC((sizeof(R) * 2) * nd * md * d, BUFFERS);
+
+     UNUSED(ii); UNUSED(ro); UNUSED(io);
+     X(transpose)(ri + ego->offset, nd, md, d, 2, buf);
+     X(ifree)(buf);
+}
+
+static void apply_slow(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int n = ego->n, m = ego->m;
+     R buf[4];
+     int move_size = (n + m) / 2;
+     char *move;
+
+     UNUSED(ii); UNUSED(ro); UNUSED(io);
+     STACK_MALLOC(char *, move, move_size);
+     X(transpose_slow)(ri + ego->offset, n, m, 2, move, move_size, buf);
+     STACK_FREE(move);
+}
+
+static int applicable(const problem *p_, const planner *plnr)
+{
+     if (DFTP(p_)) {
+          const problem_dft *p = (const problem_dft *)p_;
+	  const iodim *d = p->vecsz->dims;
+          return (1
+                  && p->ri == p->ro
+                  && p->sz->rnk == 0
+                  && p->vecsz->rnk == 2
+		  && X(transposable)(d, d+1, 1, X(imin)(d[0].is,d[0].os),
+				     p->ri, p->ii)
+		  && (!NO_UGLYP(plnr) || d[0].n == d[1].n)
+	       );
+     }
+     return 0;
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(dft-transpose-%dx%d)", ego->n, ego->m);
+}
+
+static plan *mkplan(const solver *ego, const problem *p_, planner *plnr)
+{
+     const problem_dft *p;
+     P *pln;
+     const iodim *d;
+
+     static const plan_adt padt = {
+	  X(dft_solve), X(null_awake), print, X(plan_null_destroy)
+     };
+
+     UNUSED(ego);
+
+     if (!applicable(p_, plnr))
+          return (plan *) 0;
+     p = (const problem_dft *) p_;
+
+     d = p->vecsz->dims;
+     pln = MKPLAN_DFT(P, &padt, 
+		      X(transpose_simplep)(d, d+1, 1, X(imin)(d[0].is,d[0].os),
+					   p->ri, p->ii) ? apply :
+		      (X(transpose_slowp)(d, d+1, 2) ? apply_slow : 
+		       apply_general));
+     X(transpose_dims)(d, d+1, &pln->n, &pln->m, &pln->d, &pln->nd, &pln->md);
+     pln->offset = (p->ri - p->ii == 1) ? -1 : 0;
+     pln->s0 = d[0].is;
+     pln->s1 = d[0].os;
+
+     /* (4 loads + 4 stores) * (pln->n \choose 2)
+        (FIXME? underestimate for non-square) */
+     X(ops_other)(4 * pln->n * (pln->m - 1), &pln->super.super.ops);
+     return &(pln->super.super);
+}
+
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     return MKSOLVER(S, &sadt);
+}
+
+void X(dft_vrank2_transpose_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/dft/vrank3-transpose.c b/src/fftw3/dft/vrank3-transpose.c
new file mode 100644
index 0000000..3e2157a
--- /dev/null
+++ b/src/fftw3/dft/vrank3-transpose.c
@@ -0,0 +1,207 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: vrank3-transpose.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+/* rank-0, vector-rank-3, square transposition  */
+
+#include "dft.h"
+
+/* transposition routine. TODO: optimize? */
+static void t(R *rA, R *iA, int n, int is, int js, int vn, int vs)
+{
+     int i, j, iv;
+     int im = iA - rA;
+
+     for (i = 1; i < n; ++i) {
+          for (j = 0; j < i; ++j) {
+	       R *p0 = rA + i * is + j * js;
+	       R *p1 = rA + j * is + i * js;
+               for (iv = 0; iv < vn; ++iv) {
+                    R ar = p0[0], ai = p0[im];
+                    R br = p1[0], bi = p1[im];
+                    p1[0] = ar; p1[im] = ai; p1 += vs;
+                    p0[0] = br; p0[im] = bi; p0 += vs;
+               }
+          }
+     }
+}
+
+typedef solver S;
+
+typedef struct {
+     plan_dft super;
+     int n, vl;
+     int s0, s1, vs;
+     int m;
+     int offset;
+     int nd, md, d; /* d = gcd(n,m), nd = n / d, md = m / d */
+} P;
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     UNUSED(ro);
+     UNUSED(io);
+     A(ego->n == ego->m);
+     t(ri, ii, ego->n, ego->s0, ego->s1, ego->vl, ego->vs);
+}
+
+static void apply_general(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int nd = ego->nd, md = ego->md, d = ego->d, vl = ego->vl;
+     R *buf = (R *)MALLOC((sizeof(R) * 2) * vl * nd * md * d, BUFFERS);
+
+     UNUSED(ii); UNUSED(ro); UNUSED(io);
+     X(transpose)(ri + ego->offset, nd, md, d, 2*vl, buf);
+     X(ifree)(buf);
+}
+
+static void apply_slow(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int n = ego->n, m = ego->m, vl = ego->vl;
+     R *buf = (R *)MALLOC((sizeof(R) * 4) * vl, BUFFERS);
+     int move_size = (n + m) / 2;
+     char *move;
+
+     UNUSED(ii); UNUSED(ro); UNUSED(io);
+     STACK_MALLOC(char *, move, move_size);
+     X(transpose_slow)(ri + ego->offset, n, m, 2*vl, move, move_size, buf);
+     STACK_FREE(move);
+     X(ifree)(buf);
+}
+
+static int pickdim(const tensor *s, int *pdim0, int *pdim1, int *pdim2,
+		   R *ri, R *ii)
+{
+     int dim0, dim1;
+
+     for (dim0 = 0; dim0 < s->rnk; ++dim0)
+          for (dim1 = dim0 + 1; dim1 < s->rnk; ++dim1) {
+	       int dim2 = 3 - dim0 - dim1;
+               if (s->dims[dim2].is == s->dims[dim2].os
+		   && X(transposable)(s->dims + dim0, s->dims + dim1, 
+				      s->dims[dim2].n, s->dims[dim2].is,
+				      ri, ii)) {
+                    *pdim0 = dim0;
+                    *pdim1 = dim1;
+		    *pdim2 = dim2;
+                    return 1;
+               }
+	  }
+     return 0;
+}
+
+static int applicable0(const problem *p_, int *dim0, int *dim1, int *dim2)
+{
+     if (DFTP(p_)) {
+          const problem_dft *p = (const problem_dft *)p_;
+          return (1
+                  && p->ri == p->ro
+                  && p->sz->rnk == 0
+                  && p->vecsz->rnk == 3
+                  && pickdim(p->vecsz, dim0, dim1, dim2, p->ri, p->ii)
+	       );
+     }
+     return 0;
+}
+
+static int applicable(const problem *p_, const planner *plnr, 
+		      int *dim0, int *dim1, int *dim2)
+{
+     const problem_dft *p;
+
+     if (!applicable0(p_, dim0, dim1, dim2))
+          return 0;
+
+     p = (const problem_dft *) p_;
+
+     if (NO_UGLYP(plnr))
+	  if (p->vecsz->dims[*dim2].is > X(imax)(p->vecsz->dims[*dim0].is,
+						p->vecsz->dims[*dim0].os))
+	       /* loops are in the wrong order for locality */
+	       return 0;	
+
+     if (NO_UGLYP(plnr) && p->vecsz->dims[*dim0].n != p->vecsz->dims[*dim1].n)
+	  return 0;
+
+     return 1;
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(dft-transpose-%dx%d%v)", ego->n, ego->m, ego->vl);
+}
+
+static plan *mkplan(const solver *ego, const problem *p_, planner *plnr)
+{
+     const problem_dft *p;
+     P *pln;
+     const iodim *d;
+     int dim0, dim1, dim2;
+     int vl;
+
+     static const plan_adt padt = {
+	  X(dft_solve), X(null_awake), print, X(plan_null_destroy)
+     };
+
+     UNUSED(plnr);
+     UNUSED(ego);
+
+     if (!applicable(p_, plnr, &dim0, &dim1, &dim2))
+          return (plan *) 0;
+     p = (const problem_dft *) p_;
+
+     d = p->vecsz->dims;
+     vl = d[dim2].n;
+     pln = MKPLAN_DFT(P, &padt,
+                      X(transpose_simplep)(d+dim0, d+dim1,
+					   vl, p->vecsz->dims[dim2].is,
+					   p->ri, p->ii) ? apply :
+                      (X(transpose_slowp)(d+dim0, d+dim1, 2*vl) ? apply_slow
+                       : apply_general));
+     X(transpose_dims)(d+dim0, d+dim1, 
+		       &pln->n, &pln->m, &pln->d, &pln->nd, &pln->md);
+     pln->offset = (p->ri - p->ii == 1) ? -1 : 0;
+     pln->s0 = d[dim0].is;
+     pln->s1 = d[dim0].os;
+     pln->vl = vl;
+     pln->vs = d[dim2].is; /* == os */
+
+     /* pln->vl * (4 loads + 4 stores) * (pln->n \choose 2) 
+        (FIXME? underestimate for non-square) */
+     X(ops_other)(4 * pln->vl * pln->n * (pln->m - 1), &pln->super.super.ops);
+
+     return &(pln->super.super);
+}
+
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     return MKSOLVER(S, &sadt);
+}
+
+void X(dft_vrank3_transpose_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/dft/zero.c b/src/fftw3/dft/zero.c
new file mode 100644
index 0000000..df34e0e
--- /dev/null
+++ b/src/fftw3/dft/zero.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: zero.c,v 1.1 2008/10/17 06:11:08 scuri Exp $ */
+
+#include "dft.h"
+
+/* fill a complex array with zeros. */
+static void recur(const iodim *dims, int rnk, R *ri, R *ii)
+{
+     if (rnk == RNK_MINFTY)
+          return;
+     else if (rnk == 0)
+          ri[0] = ii[0] = K(0.0);
+     else if (rnk > 0) {
+          int i, n = dims[0].n;
+          int is = dims[0].is;
+
+	  if (rnk == 1) {
+	       /* this case is redundant but faster */
+	       for (i = 0; i < n; ++i)
+		    ri[i * is] = ii[i * is] = K(0.0);
+	  } else {
+	       for (i = 0; i < n; ++i)
+		    recur(dims + 1, rnk - 1, ri + i * is, ii + i * is);
+	  }
+     }
+}
+
+
+void X(dft_zerotens)(tensor *sz, R *ri, R *ii)
+{
+     recur(sz->dims, sz->rnk, ri, ii);
+}
diff --git a/src/fftw3/kernel/align.c b/src/fftw3/kernel/align.c
new file mode 100644
index 0000000..7f475d5
--- /dev/null
+++ b/src/fftw3/kernel/align.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: align.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+#if HAVE_3DNOW
+#  define ALGN 8
+#elif HAVE_SIMD
+#  define ALGN 16
+#elif HAVE_K7
+#  define ALGN 8
+#else
+#  define ALGN (sizeof(R))
+#endif
+
+/* NONPORTABLE */
+int X(alignment_of)(R *p)
+{
+     return (int)(((uintptr_t) p) % ALGN);
+}
diff --git a/src/fftw3/kernel/alloc.c b/src/fftw3/kernel/alloc.c
new file mode 100644
index 0000000..a95e0e8
--- /dev/null
+++ b/src/fftw3/kernel/alloc.c
@@ -0,0 +1,404 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: alloc.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+#if defined(HAVE_DECL_MEMALIGN) && !HAVE_DECL_MEMALIGN
+#  if defined(HAVE_MALLOC_H)
+#    include <malloc.h>
+#  else
+extern void *memalign(size_t, size_t);
+#  endif
+#endif
+
+#if defined(HAVE_DECL_POSIX_MEMALIGN) && !HAVE_DECL_POSIX_MEMALIGN
+extern int posix_memalign(void **, size_t, size_t);
+#endif
+
+#if defined(macintosh) /* MacOS 9 */
+#  include <Multiprocessing.h>
+#endif
+
+#define real_malloc X(malloc)
+#define real_free free /* memalign and malloc use ordinary free */
+
+#if defined(WITH_OUR_MALLOC16) && (MIN_ALIGNMENT == 16)
+/* Our own 16-byte aligned malloc/free.  Assumes sizeof(void*) is a
+   power of two <= 8 and that malloc is at least sizeof(void*)-aligned.
+
+   The main reason for this routine is that, as of this writing,
+   Windows does not include any aligned allocation routines in its
+   system libraries, and instead provides an implementation with a
+   Visual C++ "Processor Pack" that you have to statically link into
+   your program.  We do not want to require users to have VC++
+   (e.g. gcc/MinGW should be fine).  Our code should be at least as good
+   as the MS _aligned_malloc, in any case, according to second-hand
+   reports of the algorithm it employs (also based on plain malloc). */
+static void *our_malloc16(size_t n)
+{
+     void *p0, *p;
+     if (!(p0 = malloc(n + 16))) return (void *) 0;
+     p = (void *) (((uintptr_t) p0 + 16) & (~((uintptr_t) 15)));
+     *((void **) p - 1) = p0;
+     return p;
+}
+static void our_free16(void *p)
+{
+     if (p) free(*((void **) p - 1));
+}
+#endif
+
+/* part of user-callable API */
+void *X(malloc)(size_t n)
+{
+     void *p;
+
+#if defined(MIN_ALIGNMENT)
+
+#  if defined(WITH_OUR_MALLOC16) && (MIN_ALIGNMENT == 16)
+     p = our_malloc16(n);
+#    undef real_free
+#    define real_free our_free16
+
+#  elif defined(HAVE_MEMALIGN)
+     p = memalign(MIN_ALIGNMENT, n);
+
+#  elif defined(HAVE_POSIX_MEMALIGN)
+     /* note: posix_memalign is broken in glibc 2.2.5: it constrains
+	the size, not the alignment, to be (power of two) * sizeof(void*).
+        The bug seems to have been fixed as of glibc 2.3.1. */
+     if (posix_memalign(&p, MIN_ALIGNMENT, n))
+	  p = (void*) 0;
+
+#  elif defined(__ICC) || defined(__INTEL_COMPILER) || defined(HAVE__MM_MALLOC)
+     /* Intel's C compiler defines _mm_malloc and _mm_free intrinsics */
+     p = (void *) _mm_malloc(n, MIN_ALIGNMENT);
+#    undef real_free
+#    define real_free _mm_free
+
+#  elif defined(_MSC_VER)
+     /* MS Visual C++ 6.0 with a "Processor Pack" supports SIMD
+	and _aligned_malloc/free (uses malloc.h) */
+     p = (void *) _aligned_malloc(n, MIN_ALIGNMENT);
+#    undef real_free
+#    define real_free _aligned_free
+
+#  elif (defined(__MACOSX__) || defined(__APPLE__)) && (MIN_ALIGNMENT <= 16)
+     /* MacOS X malloc is already 16-byte aligned */
+     p = malloc(n);
+
+#  elif defined(macintosh) /* MacOS 9 */
+     p = (void *) MPAllocateAligned(n,
+#    if MIN_ALIGNMENT == 8
+				    kMPAllocate8ByteAligned,
+#    elif MIN_ALIGNMENT == 16
+				    kMPAllocate16ByteAligned,
+#    elif MIN_ALIGNMENT == 32
+				    kMPAllocate32ByteAligned,
+#    else
+#      error "Unknown alignment for MPAllocateAligned"
+#    endif
+				    0);
+#    undef real_free
+#    define real_free MPFree
+
+#  else
+     /* Add your machine here and send a patch to fftw@fftw.org 
+        or (e.g. for Windows) configure --with-our-malloc16 */
+#    error "Don't know how to malloc() aligned memory."
+#  endif
+
+#else /* !defined(MIN_ALIGMENT) */
+     p = malloc(n);
+#endif
+
+     return p;
+}
+
+/* part of user-callable API */
+void X(free)(void *p)
+{
+     real_free(p);
+}
+
+/**********************************************************
+ *   DEBUGGING CODE
+ **********************************************************/
+#if defined(FFTW_DEBUG_MALLOC)
+
+#include <stdio.h>
+
+/*
+  debugging malloc/free. 
+ 
+  1) Initialize every malloced and freed area to random values, just
+  to make sure we are not using uninitialized pointers.
+ 
+  2) check for blocks freed twice.
+ 
+  3) Check for writes past the ends of allocated blocks
+ 
+  4) destroy contents of freed blocks in order to detect incorrect reuse.
+ 
+  5) keep track of who allocates what and report memory leaks
+ 
+  This code is a quick and dirty hack.  May be nonportable. 
+  Use at your own risk.
+ 
+*/
+
+#define MAGIC ((size_t)0xABadCafe)
+#define PAD_FACTOR 2
+#define SZ_HEADER (4 * sizeof(size_t))
+#define HASHSZ 1031
+
+static unsigned int hashaddr(void *p)
+{
+     return ((unsigned long)p) % HASHSZ;
+}
+
+struct mstat {
+     int siz;
+     int maxsiz;
+     int cnt;
+     int maxcnt;
+};
+
+static struct mstat mstat[MALLOC_WHAT_LAST];
+
+struct minfo {
+     const char *file;
+     int line;
+     size_t n;
+     void *p;
+     struct minfo *next;
+};
+
+static struct minfo *minfo[HASHSZ] = {0};
+
+#ifdef HAVE_THREADS
+int X(in_thread) = 0;
+#endif
+
+void *X(malloc_debug)(size_t n, enum malloc_tag what,
+                      const char *file, int line)
+{
+     char *p;
+     size_t i;
+     struct minfo *info;
+     struct mstat *stat = mstat + what;
+     struct mstat *estat = mstat + EVERYTHING;
+
+     if (n == 0)
+          n = 1;
+
+     if (!IN_THREAD) {
+	  stat->siz += n;
+	  if (stat->siz > stat->maxsiz)
+	       stat->maxsiz = stat->siz;
+	  estat->siz += n;
+	  if (estat->siz > estat->maxsiz)
+	       estat->maxsiz = estat->siz;
+     }
+
+     p = (char *) real_malloc(PAD_FACTOR * n + SZ_HEADER);
+     A(p);
+
+     /* store the sz in a known position */
+     ((size_t *) p)[0] = n;
+     ((size_t *) p)[1] = MAGIC;
+     ((size_t *) p)[2] = what;
+
+     /* fill with junk */
+     for (i = 0; i < PAD_FACTOR * n; i++)
+          p[i + SZ_HEADER] = (char) (i ^ 0xEF);
+
+     if (!IN_THREAD) {
+	  ++stat->cnt;
+	  ++estat->cnt;
+	  
+	  if (stat->cnt > stat->maxcnt)
+	       stat->maxcnt = stat->cnt;
+	  if (estat->cnt > estat->maxcnt)
+	       estat->maxcnt = estat->cnt;
+     }
+
+     /* skip the info we stored previously */
+     p = p + SZ_HEADER;
+
+     if (!IN_THREAD) {
+	  unsigned int h = hashaddr(p);
+	  /* record allocation in allocation list */
+	  info = (struct minfo *) malloc(sizeof(struct minfo));
+	  info->n = n;
+	  info->file = file;
+	  info->line = line;
+	  info->p = p;
+	  info->next = minfo[h];
+	  minfo[h] = info;
+     }
+
+     return (void *) p;
+}
+
+void X(ifree)(void *p)
+{
+     char *q;
+
+     A(p);
+
+     q = ((char *) p) - SZ_HEADER;
+     A(q);
+
+     {
+          size_t n = ((size_t *) q)[0];
+          size_t magic = ((size_t *) q)[1];
+          int what = ((size_t *) q)[2];
+          size_t i;
+          struct mstat *stat = mstat + what;
+          struct mstat *estat = mstat + EVERYTHING;
+
+          /* set to zero to detect duplicate free's */
+          ((size_t *) q)[0] = 0;
+
+          A(magic == MAGIC);
+          ((size_t *) q)[1] = ~MAGIC;
+
+	  if (!IN_THREAD) {
+	       stat->siz -= n;
+	       A(stat->siz >= 0);
+	       estat->siz -= n;
+	       A(estat->siz >= 0);
+	  }
+
+          /* check for writing past end of array: */
+          for (i = n; i < PAD_FACTOR * n; ++i)
+               if (q[i + SZ_HEADER] != (char) (i ^ 0xEF)) {
+                    A(0 /* array bounds overwritten */ );
+               }
+          for (i = 0; i < PAD_FACTOR * n; ++i)
+               q[i + SZ_HEADER] = (char) (i ^ 0xAD);
+
+	  if (!IN_THREAD) {
+	       --stat->cnt;
+	       --estat->cnt;
+	       
+	       A(stat->cnt >= 0);
+	       A((stat->cnt == 0 && stat->siz == 0) ||
+		 (stat->cnt > 0 && stat->siz > 0));
+	       A(estat->cnt >= 0);
+	       A((estat->cnt == 0 && estat->siz == 0) ||
+		 (estat->cnt > 0 && estat->siz > 0));
+	  }
+
+          real_free(q);
+     }
+
+     if (!IN_THREAD) {
+          /* delete minfo entry */
+	  unsigned int h = hashaddr(p);
+          struct minfo **i;
+
+          for (i = minfo + h; *i; i = &((*i)->next)) {
+               if ((*i)->p == p) {
+                    struct minfo *i0 = (*i)->next;
+                    free(*i);
+                    *i = i0;
+                    return;
+               }
+          }
+
+          A(0 /* no entry in minfo list */ );
+     }
+}
+
+void X(malloc_print_minfo)(int verbose)
+{
+     struct minfo *info;
+     int what;
+     unsigned int h;
+
+     if (verbose) {
+	  static const char *names[MALLOC_WHAT_LAST] = {
+	       "EVERYTHING",
+	       "PLANS", "SOLVERS", "PROBLEMS", "BUFFERS",
+	       "HASHT", "TENSORS", "PLANNERS", "SLVDSC", "TWIDDLES",
+	       "STRIDES", "OTHER"
+	  };
+
+	  printf("%12s %8s %8s %10s %10s\n",
+		 "what", "cnt", "maxcnt", "siz", "maxsiz");
+
+	  for (what = 0; what < MALLOC_WHAT_LAST; ++what) {
+	       struct mstat *stat = mstat + what;
+	       printf("%12s %8d %8d %10d %10d\n",
+		      names[what], stat->cnt, stat->maxcnt,
+		      stat->siz, stat->maxsiz);
+	  }
+     }
+
+     for (h = 0; h < HASHSZ; ++h) 
+	  if (minfo[h]) {
+	       printf("\nUnfreed allocations:\n");
+	       break;
+	  }
+
+     for (h = 0; h < HASHSZ; ++h) 
+	  for (info = minfo[h]; info; info = info->next) {
+	       printf("%s:%d:  %d bytes at %p\n",
+		      info->file, info->line, info->n, info->p);
+	  }
+}
+
+#else
+/**********************************************************
+ *   NON DEBUGGING CODE
+ **********************************************************/
+/* production version, no hacks */
+
+void *X(malloc_plain)(size_t n)
+{
+     void *p;
+     if (n == 0)
+          n = 1;
+     p = real_malloc(n);
+     CK(p);
+
+#ifdef MIN_ALIGMENT
+     A((((uintptr_t)p) % MIN_ALIGNMENT) == 0);
+#endif
+
+     return p;
+}
+
+void X(ifree)(void *p)
+{
+     real_free(p);
+}
+
+#endif
+
+void X(ifree0)(void *p)
+{
+     /* common pattern */
+     if (p) X(ifree)(p);
+}
diff --git a/src/fftw3/kernel/assert.c b/src/fftw3/kernel/assert.c
new file mode 100644
index 0000000..a066db0
--- /dev/null
+++ b/src/fftw3/kernel/assert.c
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: assert.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+#include "ifftw.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+void X(assertion_failed)(const char *s, int line, const char *file)
+{
+     fflush(stdout);
+     fprintf(stderr, "fftw: %s:%d: assertion failed: %s\n", file, line, s);
+     exit(EXIT_FAILURE);
+}
diff --git a/src/fftw3/kernel/awake.c b/src/fftw3/kernel/awake.c
new file mode 100644
index 0000000..7ee716f
--- /dev/null
+++ b/src/fftw3/kernel/awake.c
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: awake.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+void X(null_awake)(plan *ego, int awake)
+{
+     UNUSED(ego);
+     UNUSED(awake);
+     /* do nothing */
+}
diff --git a/src/fftw3/kernel/cycle.h b/src/fftw3/kernel/cycle.h
new file mode 100644
index 0000000..a324c2a
--- /dev/null
+++ b/src/fftw3/kernel/cycle.h
@@ -0,0 +1,420 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+/* $Id: cycle.h,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* machine-dependent cycle counters code. Needs to be inlined. */
+
+/***************************************************************************/
+/* To use the cycle counters in your code, simply #include "cycle.h" (this
+   file), and then use the functions/macros:
+
+                 ticks getticks(void);
+
+   ticks is an opaque typedef defined below, representing the current time.
+   You extract the elapsed time between two calls to gettick() via:
+
+                 double elapsed(ticks t1, ticks t0);
+
+   which returns a double-precision variable in arbitrary units.  You
+   are not expected to convert this into human units like seconds; it
+   is intended only for *comparisons* of time intervals.
+
+   (In order to use some of the OS-dependent timer routines like
+   Solaris' gethrtime, you need to paste the autoconf snippet below
+   into your configure.ac file and #include "config.h" before cycle.h,
+   or define the relevant macros manually if you are not using autoconf.)
+*/
+
+/***************************************************************************/
+/* This file uses macros like HAVE_GETHRTIME that are assumed to be
+   defined according to whether the corresponding function/type/header
+   is available on your system.  The necessary macros are most
+   conveniently defined if you are using GNU autoconf, via the tests:
+   
+   dnl ---------------------------------------------------------------------
+
+   AC_C_INLINE
+   AC_HEADER_TIME
+   AC_CHECK_HEADERS([sys/time.h c_asm.h intrinsics.h])
+
+   AC_CHECK_TYPE([hrtime_t],[AC_DEFINE(HAVE_HRTIME_T, 1, [Define to 1 if hrtime_t is defined in <sys/time.h>])],,[#if HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif])
+
+   AC_CHECK_FUNCS([gethrtime read_real_time time_base_to_time clock_gettime])
+
+   dnl Cray UNICOS _rtc() (real-time clock) intrinsic
+   AC_MSG_CHECKING([for _rtc intrinsic])
+   rtc_ok=yes
+   AC_TRY_LINK([#ifdef HAVE_INTRINSICS_H
+#include <intrinsics.h>
+#endif], [_rtc()], [AC_DEFINE(HAVE__RTC,1,[Define if you have the UNICOS _rtc() intrinsic.])], [rtc_ok=no])
+   AC_MSG_RESULT($rtc_ok)
+
+   dnl ---------------------------------------------------------------------
+*/
+
+/***************************************************************************/
+
+#if TIME_WITH_SYS_TIME
+# include <sys/time.h>
+# include <time.h>
+#else
+# if HAVE_SYS_TIME_H
+#  include <sys/time.h>
+# else
+#  include <time.h>
+# endif
+#endif
+
+#define INLINE_ELAPSED(INL) static INL double elapsed(ticks t1, ticks t0) \
+{									  \
+     return (double)(t1 - t0);						  \
+}
+
+/*----------------------------------------------------------------*/
+/* Solaris */
+#if defined(HAVE_GETHRTIME) && defined(HAVE_HRTIME_T) && !defined(HAVE_TICK_COUNTER)
+typedef hrtime_t ticks;
+
+#define getticks gethrtime
+
+INLINE_ELAPSED(inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/* AIX v. 4+ routines to read the real-time clock or time-base register */
+#if defined(HAVE_READ_REAL_TIME) && defined(HAVE_TIME_BASE_TO_TIME) && !defined(HAVE_TICK_COUNTER)
+typedef timebasestruct_t ticks;
+
+static inline ticks getticks(void)
+{
+     ticks t;
+     read_real_time(&t, TIMEBASE_SZ);
+     return t;
+}
+
+static inline double elapsed(ticks t1, ticks t0) /* time in nanoseconds */
+{
+     time_base_to_time(&t1, TIMEBASE_SZ);
+     time_base_to_time(&t0, TIMEBASE_SZ);
+     return ((t1.tb_high - t0.tb_high) * 1e9 + (t1.tb_low - t0.tb_low));
+}
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/*
+ * PowerPC ``cycle'' counter using the time base register.
+ */
+#if ((defined(__GNUC__) && (defined(__powerpc__) || defined(__ppc__))) || (defined(__MWERKS__) && defined(macintosh)))  && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     unsigned int tbl, tbu0, tbu1;
+
+     do {
+	  __asm__ __volatile__ ("mftbu %0" : "=r"(tbu0));
+	  __asm__ __volatile__ ("mftb %0" : "=r"(tbl));
+	  __asm__ __volatile__ ("mftbu %0" : "=r"(tbu1));
+     } while (tbu0 != tbu1);
+
+     return (((unsigned long long)tbu0) << 32) | tbl;
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+/*----------------------------------------------------------------*/
+/*
+ * Pentium cycle counter 
+ */
+#if (defined(__GNUC__) || defined(__ICC)) && defined(__i386__)  && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     ticks ret;
+
+     __asm__ __volatile__("rdtsc": "=A" (ret));
+     /* no input, nothing else clobbered */
+     return ret;
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/* Visual C++ -- thanks to Morten Nissov for his help with this */
+#if _MSC_VER >= 1200 && _M_IX86 >= 500 && !defined(HAVE_TICK_COUNTER)
+#include <windows.h>
+typedef LARGE_INTEGER ticks;
+#define RDTSC __asm __emit 0fh __asm __emit 031h /* hack for VC++ 5.0 */
+
+static __inline ticks getticks(void)
+{
+     ticks ret;
+
+     __asm {
+	  RDTSC
+	  mov ret.HighPart, edx
+	  mov ret.LowPart, eax
+     }
+     return ret;
+}
+
+static __inline double elapsed(ticks t1, ticks t0)
+{  
+     return (double)(t1.QuadPart - t0.QuadPart);
+}  
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/*
+ * X86-64 cycle counter
+ */
+#if defined(__GNUC__) && defined(__x86_64__)  && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     unsigned a, d; 
+     asm volatile("rdtsc" : "=a" (a), "=d" (d)); 
+     return ((ticks)a) | (((ticks)d) << 32); 
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/* Visual C++ (FIXME: how to detect compilation for x86-64?) */
+#if _MSC_VER >= 1400 && !defined(HAVE_TICK_COUNTER)
+typedef ULONG64 ticks;
+
+#define getticks __rdtsc
+
+INLINE_ELAPSED(__inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/*
+ * IA64 cycle counter
+ */
+#if defined(__GNUC__) && defined(__ia64__) && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     ticks ret;
+
+     __asm__ __volatile__ ("mov %0=ar.itc" : "=r"(ret));
+     return ret;
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/* HP/UX IA64 compiler, courtesy Teresa L. Johnson: */
+#if defined(__hpux) && defined(__ia64) && !defined(HAVE_TICK_COUNTER)
+#include <machine/sys/inline.h>
+typedef unsigned long ticks;
+
+static inline ticks getticks(void)
+{
+     ticks ret;
+
+     ret = _Asm_mov_from_ar (_AREG_ITC);
+     return ret;
+}
+
+INLINE_ELAPSED(inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/* intel's ecc compiler */
+#if defined(__ECC) && defined(__ia64__) && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long ticks;
+#include <ia64intrin.h>
+
+static __inline__ ticks getticks(void)
+{
+     return __getReg(_IA64_REG_AR_ITC);
+}
+ 
+INLINE_ELAPSED(__inline__)
+ 
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/*
+ * PA-RISC cycle counter 
+ */
+#if defined(__hppa__) || defined(__hppa) && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long ticks;
+
+#  ifdef __GNUC__
+static __inline__ ticks getticks(void)
+{
+     ticks ret;
+
+     __asm__ __volatile__("mfctl 16, %0": "=r" (ret));
+     /* no input, nothing else clobbered */
+     return ret;
+}
+#  else
+#  include <machine/inline.h>
+static inline unsigned long getticks(void)
+{
+     register ticks ret;
+     _MFCTL(16, ret);
+     return ret;
+}
+#  endif
+
+INLINE_ELAPSED(inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/* S390, courtesy of James Treacy */
+#if defined(__GNUC__) && defined(__s390__) && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     ticks cycles;
+     __asm__("stck 0(%0)" : : "a" (&(cycles)) : "memory", "cc");
+     return cycles;
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+/*----------------------------------------------------------------*/
+#if defined(__GNUC__) && defined(__alpha__) && !defined(HAVE_TICK_COUNTER)
+/*
+ * The 32-bit cycle counter on alpha overflows pretty quickly, 
+ * unfortunately.  A 1GHz machine overflows in 4 seconds.
+ */
+typedef unsigned int ticks;
+
+static __inline__ ticks getticks(void)
+{
+     unsigned long cc;
+     __asm__ __volatile__ ("rpcc %0" : "=r"(cc));
+     return (cc & 0xFFFFFFFF);
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+#if defined(__GNUC__) && defined(__sparc_v9__) && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     ticks ret;
+     __asm__("rd %%tick, %0" : "=r" (ret));
+     return ret;
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+#if defined(__DECC) && defined(__alpha) && defined(HAVE_C_ASM_H) && !defined(HAVE_TICK_COUNTER)
+#  include <c_asm.h>
+typedef unsigned int ticks;
+
+static __inline ticks getticks(void)
+{
+     unsigned long cc;
+     cc = asm("rpcc %v0");
+     return (cc & 0xFFFFFFFF);
+}
+
+INLINE_ELAPSED(__inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+/*----------------------------------------------------------------*/
+/* SGI/Irix */
+#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_SGI_CYCLE) && !defined(HAVE_TICK_COUNTER)
+typedef struct timespec ticks;
+
+static inline ticks getticks(void)
+{
+     struct timespec t;
+     clock_gettime(CLOCK_SGI_CYCLE, &t);
+     return t;
+}
+
+static inline double elapsed(ticks t1, ticks t0)
+{
+     return (double)(t1.tv_sec - t0.tv_sec) * 1.0E9 +
+	  (double)(t1.tv_nsec - t0.tv_nsec);
+}
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/* Cray UNICOS _rtc() intrinsic function */
+#if defined(HAVE__RTC) && !defined(HAVE_TICK_COUNTER)
+#ifdef HAVE_INTRINSICS_H
+#  include <intrinsics.h>
+#endif
+
+typedef long long ticks;
+
+#define getticks _rtc
+
+INLINE_ELAPSED(inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+
diff --git a/src/fftw3/kernel/debug.c b/src/fftw3/kernel/debug.c
new file mode 100644
index 0000000..63e4a22
--- /dev/null
+++ b/src/fftw3/kernel/debug.c
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: debug.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+#include "ifftw.h"
+
+#ifdef FFTW_DEBUG
+#include <stdio.h>
+
+typedef struct {
+     printer super;
+     FILE *f;
+} P_file;
+
+static void putchr_file(printer *p_, char c)
+{
+     P_file *p = (P_file *) p_;
+     fputc(c, p->f);
+}
+
+static printer *mkprinter_file(FILE *f)
+{
+     P_file *p = (P_file *) X(mkprinter)(sizeof(P_file), putchr_file, 0);
+     p->f = f;
+     return &p->super;
+}
+
+void X(debug)(const char *format, ...)
+{
+     va_list ap;
+     printer *p = mkprinter_file(stderr);
+     va_start(ap, format);
+     p->vprint(p, format, ap);
+     va_end(ap);
+     X(printer_destroy)(p);
+}
+#endif
diff --git a/src/fftw3/kernel/hash.c b/src/fftw3/kernel/hash.c
new file mode 100644
index 0000000..f12d5a4
--- /dev/null
+++ b/src/fftw3/kernel/hash.c
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "ifftw.h"
+
+unsigned X(hash)(const char *s)
+{
+     unsigned h = 0xDEADBEEFu;
+     do {
+	  h = h * 17 + (int)*s;
+     } while (*s++);
+     return h;
+}
+
diff --git a/src/fftw3/kernel/iabs.c b/src/fftw3/kernel/iabs.c
new file mode 100644
index 0000000..d2bdb7c
--- /dev/null
+++ b/src/fftw3/kernel/iabs.c
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: iabs.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+int X(iabs)(int a)
+{
+     return (int) (a < 0 ? -a : a);
+}
diff --git a/src/fftw3/kernel/ifftw.h b/src/fftw3/kernel/ifftw.h
new file mode 100644
index 0000000..0269e18
--- /dev/null
+++ b/src/fftw3/kernel/ifftw.h
@@ -0,0 +1,848 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: ifftw.h,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* FFTW internal header file */
+#ifndef __IFFTW_H__
+#define __IFFTW_H__
+
+#include "config.h"
+
+#include <stdlib.h>		/* size_t */
+#include <stdarg.h>		/* va_list */
+#include <stddef.h>             /* ptrdiff_t */
+
+#if HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+
+#if HAVE_STDINT_H
+# include <stdint.h>             /* uintptr_t, maybe */
+#endif
+
+#if HAVE_INTTYPES_H
+# include <inttypes.h>           /* uintptr_t, maybe */
+#endif
+
+/* determine precision and name-mangling scheme */
+#define CONCAT(prefix, name) prefix ## name
+#if defined(FFTW_SINGLE)
+typedef float R;
+#define X(name) CONCAT(fftwf_, name)
+#elif defined(FFTW_LDOUBLE)
+typedef long double R;
+#define X(name) CONCAT(fftwl_, name)
+#else
+typedef double R;
+#define X(name) CONCAT(fftw_, name)
+#endif
+
+/* dummy use of unused parameters to silence compiler warnings */
+#define UNUSED(x) (void)x
+
+#define FFT_SIGN (-1)  /* sign convention for forward transforms */
+
+/* get rid of that object-oriented stink: */
+#define REGISTER_SOLVER(p, s) X(solver_register)(p, s)
+
+#define STRINGIZEx(x) #x
+#define STRINGIZE(x) STRINGIZEx(x)
+
+#ifndef HAVE_K7
+#define HAVE_K7 0
+#endif
+
+#if defined(HAVE_SSE) || defined(HAVE_SSE2) || defined(HAVE_ALTIVEC) || defined(HAVE_3DNOW)
+#define HAVE_SIMD 1
+#else
+#define HAVE_SIMD 0
+#endif
+
+/* forward declarations */
+typedef struct problem_s problem;
+typedef struct plan_s plan;
+typedef struct solver_s solver;
+typedef struct planner_s planner;
+typedef struct printer_s printer;
+typedef struct scanner_s scanner;
+
+/*-----------------------------------------------------------------------*/
+/* alloca: */
+#if HAVE_SIMD
+#define MIN_ALIGNMENT 16
+#endif
+
+#ifdef HAVE_ALLOCA
+   /* use alloca if available */
+
+#ifndef alloca
+#ifdef __GNUC__
+# define alloca __builtin_alloca
+#else
+# ifdef _MSC_VER
+#  include <malloc.h>
+#  define alloca _alloca
+# else
+#  if HAVE_ALLOCA_H
+#   include <alloca.h>
+#  else
+#   ifdef _AIX
+ #pragma alloca
+#   else
+#    ifndef alloca /* predefined by HP cc +Olibcalls */
+void *alloca(size_t);
+#    endif
+#   endif
+#  endif
+# endif
+#endif
+#endif
+
+#  ifdef MIN_ALIGNMENT
+#    define STACK_MALLOC(T, p, x)				\
+     {								\
+         p = (T)alloca((x) + MIN_ALIGNMENT);			\
+         p = (T)(((uintptr_t)p + (MIN_ALIGNMENT - 1)) &	\
+               (~(uintptr_t)(MIN_ALIGNMENT - 1)));		\
+     }
+#    define STACK_FREE(x) 
+#  else /* HAVE_ALLOCA && !defined(MIN_ALIGNMENT) */
+#    define STACK_MALLOC(T, p, x) p = (T)alloca(x) 
+#    define STACK_FREE(x) 
+#  endif
+
+#else /* ! HAVE_ALLOCA */
+   /* use malloc instead of alloca */
+#  define STACK_MALLOC(T, p, x) p = (T)MALLOC(x, OTHER)
+#  define STACK_FREE(x) X(ifree)(x)
+#endif /* ! HAVE_ALLOCA */
+
+/*-----------------------------------------------------------------------*/
+/* define uintptr_t if it is not already defined */
+
+#ifndef HAVE_UINTPTR_T
+#  if SIZEOF_VOID_P == 0
+#    error sizeof void* is unknown!
+#  elif SIZEOF_UNSIGNED_INT == SIZEOF_VOID_P
+     typedef unsigned int uintptr_t;
+#  elif SIZEOF_UNSIGNED_LONG == SIZEOF_VOID_P
+     typedef unsigned long uintptr_t;
+#  elif SIZEOF_UNSIGNED_LONG_LONG == SIZEOF_VOID_P
+     typedef unsigned long long uintptr_t;
+#  else
+#    error no unsigned integer type matches void* sizeof!
+#  endif
+#endif
+
+/*-----------------------------------------------------------------------*/
+/* assert.c: */
+extern void X(assertion_failed)(const char *s, int line, const char *file);
+
+/* always check */
+#define CK(ex)						 \
+      (void)((ex) || (X(assertion_failed)(#ex, __LINE__, __FILE__), 0))
+
+#ifdef FFTW_DEBUG
+/* check only if debug enabled */
+#define A(ex)						 \
+      (void)((ex) || (X(assertion_failed)(#ex, __LINE__, __FILE__), 0))
+#else
+#define A(ex) /* nothing */
+#endif
+
+extern void X(debug)(const char *format, ...);
+#define D X(debug)
+
+/*-----------------------------------------------------------------------*/
+/* alloc.c: */
+
+/* objects allocated by malloc, for statistical purposes */
+enum malloc_tag {
+     EVERYTHING,
+     PLANS,
+     SOLVERS,
+     PROBLEMS,
+     BUFFERS,
+     HASHT,
+     TENSORS,
+     PLANNERS,
+     SLVDESCS,
+     TWIDDLES,
+     STRIDES,
+     OTHER,
+     MALLOC_WHAT_LAST		/* must be last */
+};
+
+extern void X(ifree)(void *ptr);
+extern void X(ifree0)(void *ptr);
+
+#ifdef FFTW_DEBUG_MALLOC
+
+extern void *X(malloc_debug)(size_t n, enum malloc_tag what,
+			     const char *file, int line);
+#define MALLOC(n, what) X(malloc_debug)(n, what, __FILE__, __LINE__)
+#define NATIVE_MALLOC(n, what) MALLOC(n, what)
+void X(malloc_print_minfo)(int vrbose);
+
+#else /* ! FFTW_DEBUG_MALLOC */
+
+extern void *X(malloc_plain)(size_t sz);
+#define MALLOC(n, what)  X(malloc_plain)(n)
+#define NATIVE_MALLOC(n, what) malloc(n)
+
+#endif
+
+#if defined(FFTW_DEBUG) && defined(FFTW_DEBUG_MALLOC) && defined(HAVE_THREADS)
+extern int X(in_thread);
+#  define IN_THREAD X(in_thread)
+#  define THREAD_ON { int in_thread_save = X(in_thread); X(in_thread) = 1
+#  define THREAD_OFF X(in_thread) = in_thread_save; }
+#else
+#  define IN_THREAD 0
+#  define THREAD_ON 
+#  define THREAD_OFF 
+#endif
+
+/*-----------------------------------------------------------------------*/
+/* ops.c: */
+/*
+ * ops counter.  The total number of additions is add + fma
+ * and the total number of multiplications is mul + fma.
+ * Total flops = add + mul + 2 * fma
+ */
+typedef struct {
+     double add;
+     double mul;
+     double fma;
+     double other;
+} opcnt;
+
+void X(ops_zero)(opcnt *dst);
+void X(ops_other)(int o, opcnt *dst);
+void X(ops_cpy)(const opcnt *src, opcnt *dst);
+
+void X(ops_add)(const opcnt *a, const opcnt *b, opcnt *dst);
+void X(ops_add2)(const opcnt *a, opcnt *dst);
+
+/* dst = m * a + b */
+void X(ops_madd)(int m, const opcnt *a, const opcnt *b, opcnt *dst);
+
+/* dst += m * a */
+void X(ops_madd2)(int m, const opcnt *a, opcnt *dst);
+
+
+/*-----------------------------------------------------------------------*/
+/* minmax.c: */
+int X(imax)(int a, int b);
+int X(imin)(int a, int b);
+
+/*-----------------------------------------------------------------------*/
+/* iabs.c: */
+int X(iabs)(int a);
+
+/*-----------------------------------------------------------------------*/
+/* md5.c */
+
+#if SIZEOF_UNSIGNED_INT >= 4
+typedef unsigned int md5uint;
+#else
+typedef unsigned long md5uint; /* at least 32 bits as per C standard */
+#endif
+
+typedef md5uint md5sig[4];
+
+typedef struct {
+     md5sig s; /* state and signature */
+
+     /* fields not meant to be used outside md5.c: */
+     unsigned char c[64]; /* stuff not yet processed */
+     unsigned l;  /* total length.  Should be 64 bits long, but this is
+		     good enough for us */
+} md5;
+
+void X(md5begin)(md5 *p);
+void X(md5putb)(md5 *p, const void *d_, int len);
+void X(md5puts)(md5 *p, const char *s);
+void X(md5putc)(md5 *p, unsigned char c);
+void X(md5int)(md5 *p, int i);
+void X(md5unsigned)(md5 *p, unsigned i);
+void X(md5ptrdiff)(md5 *p, ptrdiff_t d);
+void X(md5end)(md5 *p);
+
+/*-----------------------------------------------------------------------*/
+/* tensor.c: */
+#define STRUCT_HACK_KR
+#undef STRUCT_HACK_C99
+
+typedef struct {
+     int n;
+     int is;			/* input stride */
+     int os;			/* output stride */
+} iodim;
+
+typedef struct {
+     int rnk;
+#if defined(STRUCT_HACK_KR)
+     iodim dims[1];
+#elif defined(STRUCT_HACK_C99)
+     iodim dims[];
+#else
+     iodim *dims;
+#endif
+} tensor;
+
+/*
+  Definition of rank -infinity.
+  This definition has the property that if you want rank 0 or 1,
+  you can simply test for rank <= 1.  This is a common case.
+ 
+  A tensor of rank -infinity has size 0.
+*/
+#define RNK_MINFTY  ((int)(((unsigned) -1) >> 1))
+#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY)
+
+typedef enum { INPLACE_IS, INPLACE_OS } inplace_kind;
+
+tensor *X(mktensor)(int rnk);
+tensor *X(mktensor_0d)(void);
+tensor *X(mktensor_1d)(int n, int is, int os);
+tensor *X(mktensor_2d)(int n0, int is0, int os0,
+                      int n1, int is1, int os1);
+int X(tensor_sz)(const tensor *sz);
+void X(tensor_md5)(md5 *p, const tensor *t);
+int X(tensor_max_index)(const tensor *sz);
+int X(tensor_min_istride)(const tensor *sz);
+int X(tensor_min_ostride)(const tensor *sz);
+int X(tensor_min_stride)(const tensor *sz);
+int X(tensor_inplace_strides)(const tensor *sz);
+int X(tensor_inplace_strides2)(const tensor *a, const tensor *b);
+tensor *X(tensor_copy)(const tensor *sz);
+int X(tensor_kosherp)(const tensor *x);
+
+tensor *X(tensor_copy_inplace)(const tensor *sz, inplace_kind k);
+tensor *X(tensor_copy_except)(const tensor *sz, int except_dim);
+tensor *X(tensor_copy_sub)(const tensor *sz, int start_dim, int rnk);
+tensor *X(tensor_compress)(const tensor *sz);
+tensor *X(tensor_compress_contiguous)(const tensor *sz);
+tensor *X(tensor_append)(const tensor *a, const tensor *b);
+void X(tensor_split)(const tensor *sz, tensor **a, int a_rnk, tensor **b);
+int X(tensor_tornk1)(const tensor *t, int *n, int *is, int *os);
+void X(tensor_destroy)(tensor *sz);
+void X(tensor_destroy2)(tensor *a, tensor *b);
+void X(tensor_destroy4)(tensor *a, tensor *b, tensor *c, tensor *d);
+void X(tensor_print)(const tensor *sz, printer *p);
+int X(dimcmp)(const iodim *a, const iodim *b);
+
+/*-----------------------------------------------------------------------*/
+/* problem.c: */
+typedef struct {
+     void (*hash) (const problem *ego, md5 *p);
+     void (*zero) (const problem *ego);
+     void (*print) (problem *ego, printer *p);
+     void (*destroy) (problem *ego);
+} problem_adt;
+
+struct problem_s {
+     const problem_adt *adt;
+};
+
+problem *X(mkproblem)(size_t sz, const problem_adt *adt);
+void X(problem_destroy)(problem *ego);
+
+/*-----------------------------------------------------------------------*/
+/* print.c */
+struct printer_s {
+     void (*print)(printer *p, const char *format, ...);
+     void (*vprint)(printer *p, const char *format, va_list ap);
+     void (*putchr)(printer *p, char c);
+     void (*cleanup)(printer *p);
+     int indent;
+     int indent_incr;
+};
+
+printer *X(mkprinter)(size_t size, 
+		      void (*putchr)(printer *p, char c),
+		      void (*cleanup)(printer *p));
+void X(printer_destroy)(printer *p);
+
+/*-----------------------------------------------------------------------*/
+/* scan.c */
+struct scanner_s {
+     int (*scan)(scanner *sc, const char *format, ...);
+     int (*vscan)(scanner *sc, const char *format, va_list ap);
+     int (*getchr)(scanner *sc);
+     int ungotc;
+};
+
+scanner *X(mkscanner)(size_t size, int (*getchr)(scanner *sc));
+void X(scanner_destroy)(scanner *sc);
+
+/*-----------------------------------------------------------------------*/
+/* plan.c: */
+typedef struct {
+     void (*solve)(const plan *ego, const problem *p);
+     void (*awake)(plan *ego, int flag);
+     void (*print)(const plan *ego, printer *p);
+     void (*destroy)(plan *ego);
+} plan_adt;
+
+struct plan_s {
+     const plan_adt *adt;
+     int awake_refcnt;
+     opcnt ops;
+     double pcost;
+};
+
+plan *X(mkplan)(size_t size, const plan_adt *adt);
+void X(plan_destroy_internal)(plan *ego);
+void X(plan_awake)(plan *ego, int flag);
+#define AWAKE(plan, flag) X(plan_awake)(plan, flag)
+void X(plan_null_destroy)(plan *ego);
+
+/*-----------------------------------------------------------------------*/
+/* solver.c: */
+typedef struct {
+     plan *(*mkplan)(const solver *ego, const problem *p, planner *plnr);
+} solver_adt;
+
+struct solver_s {
+     const solver_adt *adt;
+     int refcnt;
+};
+
+solver *X(mksolver)(size_t size, const solver_adt *adt);
+void X(solver_use)(solver *ego);
+void X(solver_destroy)(solver *ego);
+void X(solver_register)(planner *plnr, solver *s);
+
+/* shorthand */
+#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt)
+
+/*-----------------------------------------------------------------------*/
+/* planner.c */
+
+typedef struct slvdesc_s {
+     solver *slv;
+     const char *reg_nam;
+     unsigned nam_hash;
+     int reg_id;
+} slvdesc;
+
+typedef struct solution_s solution; /* opaque */
+
+/* values for problem_flags: */
+enum { 
+     DESTROY_INPUT = 0x1,
+     NO_SIMD = 0x2,
+     CONSERVE_MEMORY = 0x4,
+     NO_DHT_R2HC = 0x8
+};
+
+#define DESTROY_INPUTP(plnr) ((plnr)->problem_flags & DESTROY_INPUT)
+#define NO_SIMDP(plnr) ((plnr)->problem_flags & NO_SIMD)
+#define CONSERVE_MEMORYP(plnr) ((plnr)->problem_flags & CONSERVE_MEMORY)
+#define NO_DHT_R2HCP(plnr) ((plnr)->problem_flags & NO_DHT_R2HC)
+
+/* values for planner_flags: */
+enum {
+     /* impatience flags  */
+
+     BELIEVE_PCOST = 0x1,
+     DFT_R2HC_ICKY = 0x2,
+     NONTHREADED_ICKY = 0x4,
+     NO_BUFFERING = 0x8,
+     NO_EXHAUSTIVE = 0x10,
+     NO_INDIRECT_OP = 0x20,
+     NO_LARGE_GENERIC = 0x40,
+     NO_RANK_SPLITS = 0x80,
+     NO_VRANK_SPLITS = 0x100,
+     NO_VRECURSE = 0x200,
+     
+     /* flags that control the search */
+     NO_UGLY = 0x400,  /* avoid plans we are 99% sure are suboptimal */
+     NO_SEARCH = 0x800,  /* avoid searching altogether---use wisdom entries 
+			    only */
+
+     ESTIMATE = 0x1000,
+     IMPATIENCE_FLAGS = (ESTIMATE | (ESTIMATE - 1)),
+     
+     BLESSING = 0x4000,  /* save this entry */
+     H_VALID = 0x8000,    /* valid hastable entry */
+     NONIMPATIENCE_FLAGS = BLESSING
+};
+
+#define BELIEVE_PCOSTP(plnr) ((plnr)->planner_flags & BELIEVE_PCOST)
+#define DFT_R2HC_ICKYP(plnr) ((plnr)->planner_flags & DFT_R2HC_ICKY)
+#define ESTIMATEP(plnr) ((plnr)->planner_flags & ESTIMATE)
+#define NONTHREADED_ICKYP(plnr) (((plnr)->planner_flags & NONTHREADED_ICKY) \
+                                  && (plnr)->nthr > 1)
+#define NO_BUFFERINGP(plnr) ((plnr)->planner_flags & NO_BUFFERING)
+#define NO_EXHAUSTIVEP(plnr) ((plnr)->planner_flags & NO_EXHAUSTIVE)
+#define NO_INDIRECT_OP_P(plnr) ((plnr)->planner_flags & NO_INDIRECT_OP)
+#define NO_LARGE_GENERICP(plnr) ((plnr)->planner_flags & NO_LARGE_GENERIC)
+#define NO_RANK_SPLITSP(plnr) ((plnr)->planner_flags & NO_RANK_SPLITS)
+#define NO_UGLYP(plnr) ((plnr)->planner_flags & NO_UGLY)
+#define NO_SEARCHP(plnr) ((plnr)->planner_flags & NO_SEARCH)
+#define NO_VRANK_SPLITSP(plnr) ((plnr)->planner_flags & NO_VRANK_SPLITS)
+#define NO_VRECURSEP(plnr) ((plnr)->planner_flags & NO_VRECURSE)
+
+typedef enum { FORGET_ACCURSED, FORGET_EVERYTHING } amnesia;
+
+typedef struct {
+     void (*register_solver)(planner *ego, solver *s);
+     plan *(*mkplan)(planner *ego, problem *p);
+     void (*forget)(planner *ego, amnesia a);
+     void (*exprt)(planner *ego, printer *p); /* ``export'' is a reserved
+						 word in C++. */
+     int (*imprt)(planner *ego, scanner *sc);
+} planner_adt;
+
+struct planner_s {
+     const planner_adt *adt;
+     void (*hook)(plan *pln, const problem *p, int optimalp);
+
+     /* solver descriptors */
+     slvdesc *slvdescs;
+     unsigned nslvdesc, slvdescsiz;
+     const char *cur_reg_nam;
+     int cur_reg_id;
+
+     /* hash table of solutions */
+     solution *solutions;
+     unsigned hashsiz, nelem;
+
+     int nthr;
+     unsigned problem_flags;
+     unsigned short planner_flags; /* matches type of solution.flags in
+				      planner.c */
+     /* various statistics */
+     int nplan;    /* number of plans evaluated */
+     double pcost, epcost; /* total pcost of measured/estimated plans */
+     int nprob;    /* number of problems evaluated */
+     int lookup, succ_lookup, lookup_iter;
+     int insert, insert_iter, insert_unknown;
+     int nrehash;
+};
+
+planner *X(mkplanner)(void);
+void X(planner_destroy)(planner *ego);
+
+#ifdef FFTW_DEBUG
+void X(planner_dump)(planner *ego, int vrbose);
+#endif
+
+/*
+  Iterate over all solvers.   Read:
+ 
+  @article{ baker93iterators,
+  author = "Henry G. Baker, Jr.",
+  title = "Iterators: Signs of Weakness in Object-Oriented Languages",
+  journal = "{ACM} {OOPS} Messenger",
+  volume = "4",
+  number = "3",
+  pages = "18--25"
+  }
+*/
+#define FORALL_SOLVERS(ego, s, p, what)			\
+{							\
+     unsigned _cnt;					\
+     for (_cnt = 0; _cnt < ego->nslvdesc; ++_cnt) {	\
+	  slvdesc *p = ego->slvdescs + _cnt;		\
+	  solver *s = p->slv;				\
+	  what;						\
+     }							\
+}
+
+/* make plan, destroy problem */
+plan *X(mkplan_d)(planner *ego, problem *p);
+
+/*-----------------------------------------------------------------------*/
+/* stride.c: */
+
+/* If PRECOMPUTE_ARRAY_INDICES is defined, precompute all strides. */
+#if (defined(__i386__) || _M_IX86 >= 500) && !HAVE_K7 && !defined(FFTW_LDOUBLE)
+#define PRECOMPUTE_ARRAY_INDICES
+#endif
+
+#ifdef PRECOMPUTE_ARRAY_INDICES
+typedef int *stride;
+#define WS(stride, i)  (stride[i])
+extern stride X(mkstride)(int n, int s);
+void X(stride_destroy)(stride p);
+
+#else
+
+typedef int stride;
+#define WS(stride, i)  (stride * i)
+#define fftwf_mkstride(n, stride) stride
+#define fftw_mkstride(n, stride) stride
+#define fftwl_mkstride(n, stride) stride
+#define fftwf_stride_destroy(p) ((void) p)
+#define fftw_stride_destroy(p) ((void) p)
+#define fftwl_stride_destroy(p) ((void) p)
+
+#endif /* PRECOMPUTE_ARRAY_INDICES */
+
+/*-----------------------------------------------------------------------*/
+/* solvtab.c */
+
+struct solvtab_s { void (*reg)(planner *); const char *reg_nam; };
+typedef struct solvtab_s solvtab[];
+void X(solvtab_exec)(solvtab tbl, planner *p);
+#define SOLVTAB(s) { s, STRINGIZE(s) }
+#define SOLVTAB_END { 0, 0 }
+
+/*-----------------------------------------------------------------------*/
+/* pickdim.c */
+int X(pickdim)(int which_dim, const int *buddies, int nbuddies,
+	       const tensor *sz, int oop, int *dp);
+
+/*-----------------------------------------------------------------------*/
+/* twiddle.c */
+/* little language to express twiddle factors computation */
+enum { TW_COS = 0, TW_SIN = 1, TW_TAN = 2, TW_NEXT = 3,
+       TW_FULL = 4, TW_GENERIC = 5 };
+
+typedef struct {
+     unsigned char op;
+     unsigned char v;
+     short i;
+} tw_instr;
+
+typedef struct twid_s {
+     R *W;                     /* array of twiddle factors */
+     int n, r, m;                /* transform order, radix, # twiddle rows */
+     int refcnt;
+     const tw_instr *instr;
+     struct twid_s *cdr;
+} twid;
+
+void X(mktwiddle)(twid **pp, const tw_instr *instr, int n, int r, int m);
+void X(twiddle_destroy)(twid **pp);
+int X(twiddle_length)(int r, const tw_instr *p);
+void X(twiddle_awake)(int flg, twid **pp, 
+		      const tw_instr *instr, int n, int r, int m);
+
+/*-----------------------------------------------------------------------*/
+/* trig.c */
+#ifdef FFTW_LDOUBLE
+typedef long double trigreal;
+#else
+typedef double trigreal;
+#endif
+
+extern trigreal X(cos2pi)(int, int);
+extern trigreal X(sin2pi)(int, int);
+extern trigreal X(tan2pi)(int, int);
+extern trigreal X(sincos)(trigreal m, trigreal n, int sinp);
+
+/*-----------------------------------------------------------------------*/
+/* primes.c: */
+
+#if defined(FFTW_ENABLE_UNSAFE_MULMOD)
+#  define MULMOD(x,y,p) (((x) * (y)) % (p))
+#elif ((SIZEOF_INT != 0) && (SIZEOF_LONG >= 2 * SIZEOF_INT))
+#  define MULMOD(x,y,p) ((int) ((((long) (x)) * ((long) (y))) % ((long) (p))))
+#elif ((SIZEOF_INT != 0) && (SIZEOF_LONG_LONG >= 2 * SIZEOF_INT))
+#  define MULMOD(x,y,p) ((int) ((((long long) (x)) * ((long long) (y))) \
+				 % ((long long) (p))))
+#elif defined(_MSC_VER)
+#  define MULMOD(x,y,p) ((int) ((((__int64) (x)) * ((__int64) (y))) \
+                                 % ((__int64) (p))))
+#else /* 'long long' unavailable */
+#  define SAFE_MULMOD 1
+int X(safe_mulmod)(int x, int y, int p);
+#  define MULMOD(x,y,p) X(safe_mulmod)(x,y,p)
+#endif
+
+int X(power_mod)(int n, int m, int p);
+int X(find_generator)(int p);
+int X(first_divisor)(int n);
+int X(is_prime)(int n);
+int X(next_prime)(int n);
+
+#define GENERIC_MIN_BAD 71 /* min prime for which generic becomes bad */
+
+/*-----------------------------------------------------------------------*/
+/* rader.c: */
+typedef struct rader_tls rader_tl;
+
+void X(rader_tl_insert)(int k1, int k2, int k3, R *W, rader_tl **tl);
+R *X(rader_tl_find)(int k1, int k2, int k3, rader_tl *t);
+void X(rader_tl_delete)(R *W, rader_tl **tl);
+
+/*-----------------------------------------------------------------------*/
+/* transpose.c: */
+
+void X(transpose)(R *A, int n, int m, int d, int N, R *buf);
+void X(transpose_slow)(R *a, int nx, int ny, int N,
+		       char *move, int move_size, R *buf);
+int X(transposable)(const iodim *a, const iodim *b,
+		    int vl, int s, R *ri, R *ii);
+void X(transpose_dims)(const iodim *a, const iodim *b,
+                       int *n, int *m, int *d, int *nd, int *md);
+int X(transpose_simplep)(const iodim *a, const iodim *b, int vl, int s,
+			 R *ri, R *ii);
+int X(transpose_slowp)(const iodim *a, const iodim *b, int N);
+
+/*-----------------------------------------------------------------------*/
+/* misc stuff */
+void X(null_awake)(plan *ego, int awake);
+int X(square)(int x);
+double X(measure_execution_time)(plan *pln, const problem *p);
+int X(alignment_of)(R *p);
+unsigned X(hash)(const char *s);
+int X(compute_nbuf)(int n, int vl, int nbuf, int maxbufsz);
+int X(ct_uglyp)(int min_n, int n, int r);
+
+#if HAVE_SIMD
+R *X(taint)(R *p, int s);
+R *X(join_taint)(R *p1, R *p2);
+#define TAINT(p, s) X(taint)(p, s)
+#define UNTAINT(p) ((R *) (((uintptr_t) (p)) & ~(uintptr_t)3))
+#define TAINTOF(p) (((uintptr_t)(p)) & 3)
+#define JOIN_TAINT(p1, p2) X(join_taint)(p1, p2)
+#else
+#define TAINT(p, s) (p)
+#define UNTAINT(p) (p)
+#define TAINTOF(p) 0
+#define JOIN_TAINT(p1, p2) p1
+#endif
+
+#ifdef FFTW_DEBUG_ALIGNMENT
+#  define ASSERT_ALIGNED_DOUBLE {		\
+     double __foo;				\
+     CK(!(((uintptr_t) &__foo) & 0x7));		\
+}
+#else
+#  define ASSERT_ALIGNED_DOUBLE 
+#endif /* FFTW_DEBUG_ALIGNMENT */
+
+
+
+/*-----------------------------------------------------------------------*/
+/* macros used in codelets to reduce source code size */
+
+typedef R E;  /* internal precision of codelets. */
+
+#ifdef FFTW_LDOUBLE
+#  define K(x) ((E) x##L)
+#else
+#  define K(x) ((E) x)
+#endif
+#define DK(name, value) const E name = K(value)
+
+/* FMA macros */
+
+#if defined(__GNUC__) && (defined(__powerpc__) || defined(__ppc__))
+/* this peculiar coding seems to do the right thing on all of
+   gcc-2.95, gcc-3.1, and gcc-3.2.  
+
+   The obvious expression a * b + c does not work.  If both x = a * b
+   + c and y = a * b - c appear in the source, gcc computes t = a * b,
+   x = t + c, y = t - c, thus destroying the fma.
+*/
+static __inline__ E FMA(E a, E b, E c)
+{
+     E x = a * b;
+     x = x + c;
+     return x;
+}
+
+static __inline__ E FMS(E a, E b, E c)
+{
+     E x = a * b;
+     x = x - c;
+     return x;
+}
+
+static __inline__ E FNMA(E a, E b, E c)
+{
+     E x = a * b;
+     x = - (x + c);
+     return x;
+}
+
+static __inline__ E FNMS(E a, E b, E c)
+{
+     E x = a * b;
+     x = - (x - c);
+     return x;
+}
+#else
+#define FMA(a, b, c) (((a) * (b)) + (c))
+#define FMS(a, b, c) (((a) * (b)) - (c))
+#define FNMA(a, b, c) (- (((a) * (b)) + (c)))
+#define FNMS(a, b, c) ((c) - ((a) * (b)))
+#endif
+
+
+/* stack-alignment hackery */
+#if defined(__GNUC__) && defined(__i386__)
+/*
+ * horrible hack to align the stack to a 16-byte boundary.
+ *
+ * We assume a gcc version >= 2.95 so that
+ * -mpreferred-stack-boundary works.  Otherwise, all bets are
+ * off.  However, -mpreferred-stack-boundary does not create a
+ * stack alignment, but it only preserves it.  Unfortunately,
+ * many versions of libc on linux call main() with the wrong
+ * initial stack alignment, with the result that the code is now
+ * pessimally aligned instead of having a 50% chance of being
+ * correct.
+ */
+
+#define WITH_ALIGNED_STACK(what)				\
+{								\
+     /*								\
+      * Use alloca to allocate some memory on the stack.	\
+      * This alerts gcc that something funny is going		\
+      * on, so that it does not omit the frame pointer		\
+      * etc.							\
+      */							\
+     (void)__builtin_alloca(16);				\
+								\
+     /*								\
+      * Now align the stack pointer				\
+      */							\
+     __asm__ __volatile__ ("andl $-16, %esp");			\
+								\
+     what							\
+}
+#endif
+
+#ifdef __ICC /* Intel's compiler for ia32 */
+#define WITH_ALIGNED_STACK(what)				\
+{								\
+     /*								\
+      * Simply calling alloca seems to do the right thing.	\
+      * The size of the allocated block seems to be irrelevant.	\
+      */							\
+     _alloca(16);						\
+     what							\
+}
+#endif
+
+#ifndef WITH_ALIGNED_STACK
+#define WITH_ALIGNED_STACK(what) what
+#endif
+
+#endif /* __IFFTW_H__ */
diff --git a/src/fftw3/kernel/kbuffered.c b/src/fftw3/kernel/kbuffered.c
new file mode 100644
index 0000000..d20403f
--- /dev/null
+++ b/src/fftw3/kernel/kbuffered.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* routines shared by the various buffered solvers */
+
+#include "ifftw.h"
+
+int X(compute_nbuf)(int n, int vl, int nbuf, int maxbufsz)
+{
+     int i; 
+
+     if (nbuf * n > maxbufsz)
+          nbuf = X(imax)((int)1, maxbufsz / n);
+
+     /*
+      * Look for a buffer number (not too big) that divides the
+      * vector length, in order that we only need one child plan:
+      */
+     for (i = nbuf; i < vl && i < 2 * nbuf; ++i)
+          if (vl % i == 0)
+               return i;
+
+     /* whatever... */
+     nbuf = X(imin)(nbuf, vl);
+     return nbuf;
+}
+
diff --git a/src/fftw3/kernel/kct.c b/src/fftw3/kernel/kct.c
new file mode 100644
index 0000000..ba3bf68
--- /dev/null
+++ b/src/fftw3/kernel/kct.c
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* common routines for Cooley-Tukey algorithms */
+
+#include "ifftw.h"
+
+#define POW2P(n) (((n) > 0) && (((n) & ((n) - 1)) == 0))
+
+/* TRUE if radix-r is ugly for size n */
+int X(ct_uglyp)(int min_n, int n, int r)
+{
+     return (n <= min_n) || (POW2P(n) && (n / r) <= 4);
+}
diff --git a/src/fftw3/kernel/kplan.c b/src/fftw3/kernel/kplan.c
new file mode 100644
index 0000000..4c17107
--- /dev/null
+++ b/src/fftw3/kernel/kplan.c
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: kplan.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+/* "Plan: To bother about the best method of accomplishing an
+   accidental result."  (Ambrose Bierce, The Enlarged Devil's
+   Dictionary). */
+
+plan *X(mkplan)(size_t size, const plan_adt *adt)
+{
+     plan *p = (plan *)MALLOC(size, PLANS);
+
+     A(adt->destroy);
+     p->awake_refcnt = 0;
+     p->adt = adt;
+     X(ops_zero)(&p->ops);
+     p->pcost = 0.0;
+     
+     return p;
+}
+
+/*
+ * destroy a plan
+ */
+void X(plan_destroy_internal)(plan *ego)
+{
+     if (ego) {
+	  if (ego->awake_refcnt > 0)
+	       ego->adt->awake(ego, 0);
+          ego->adt->destroy(ego);
+	  X(ifree)(ego);
+     }
+}
+
+/* dummy destroy routine for plans with no local state */
+void X(plan_null_destroy)(plan *ego)
+{
+     UNUSED(ego);
+     /* nothing */
+}
+
+void X(plan_awake)(plan *ego, int flag)
+{
+     if (flag) {
+	  if (!ego->awake_refcnt)
+	       ego->adt->awake(ego, flag);
+	  ++ego->awake_refcnt;
+     } else {
+	  --ego->awake_refcnt;
+	  if (!ego->awake_refcnt)
+	       ego->adt->awake(ego, flag);
+     }
+}
+
diff --git a/src/fftw3/kernel/kproblem.c b/src/fftw3/kernel/kproblem.c
new file mode 100644
index 0000000..4686692
--- /dev/null
+++ b/src/fftw3/kernel/kproblem.c
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: kproblem.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+/* constructor */
+problem *X(mkproblem)(size_t sz, const problem_adt *adt)
+{
+     problem *p = (problem *)MALLOC(sz, PROBLEMS);
+
+     p->adt = adt;
+     return p;
+}
+
+/* destructor */
+void X(problem_destroy)(problem *ego)
+{
+     if (ego)
+	  ego->adt->destroy(ego);
+}
+
diff --git a/src/fftw3/kernel/krader.c b/src/fftw3/kernel/krader.c
new file mode 100644
index 0000000..354edfc
--- /dev/null
+++ b/src/fftw3/kernel/krader.c
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "ifftw.h"
+
+/*
+  common routines for Rader solvers 
+*/
+
+
+/* shared twiddle and omega lists, keyed by two/three integers. */
+struct rader_tls {
+     int k1, k2, k3;
+     R *W;
+     int refcnt;
+     rader_tl *cdr; 
+};
+
+void X(rader_tl_insert)(int k1, int k2, int k3, R *W, rader_tl **tl)
+{
+     rader_tl *t = (rader_tl *) MALLOC(sizeof(rader_tl), TWIDDLES);
+     t->k1 = k1; t->k2 = k2; t->k3 = k3; t->W = W;
+     t->refcnt = 1; t->cdr = *tl; *tl = t;
+}
+
+R *X(rader_tl_find)(int k1, int k2, int k3, rader_tl *t)
+{
+     while (t && (t->k1 != k1 || t->k2 != k2 || t->k3 != k3))
+	  t = t->cdr;
+     if (t) {
+	  ++t->refcnt;
+	  return t->W;
+     } else 
+	  return 0;
+}
+
+void X(rader_tl_delete)(R *W, rader_tl **tl)
+{
+     if (W) {
+	  rader_tl **tp, *t;
+
+	  for (tp = tl; (t = *tp) && t->W != W; tp = &t->cdr)
+	       ;
+
+	  if (t && --t->refcnt <= 0) {
+	       *tp = t->cdr;
+	       X(ifree)(t->W);
+	       X(ifree)(t);
+	  }
+     }
+}
diff --git a/src/fftw3/kernel/md5-1.c b/src/fftw3/kernel/md5-1.c
new file mode 100644
index 0000000..a2e87a1
--- /dev/null
+++ b/src/fftw3/kernel/md5-1.c
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "ifftw.h"
+
+
+void X(md5putb)(md5 *p, const void *d_, int len)
+{
+     int i;
+     const unsigned char *d = (const unsigned char *)d_;
+     for (i = 0; i < len; ++i)
+	  X(md5putc)(p, d[i]);
+}
+
+void X(md5puts)(md5 *p, const char *s)
+{
+     /* also hash final '\0' */
+     do {
+	  X(md5putc)(p, *s);
+     } while(*s++);
+}
+
+void X(md5int)(md5 *p, int i)
+{
+     X(md5putb)(p, &i, sizeof(i));
+}
+
+void X(md5unsigned)(md5 *p, unsigned i)
+{
+     X(md5putb)(p, &i, sizeof(i));
+}
+
+void X(md5ptrdiff)(md5 *p, ptrdiff_t d)
+{
+     X(md5putb)(p, &d, sizeof(d));
+}
+
diff --git a/src/fftw3/kernel/md5.c b/src/fftw3/kernel/md5.c
new file mode 100644
index 0000000..edbd811
--- /dev/null
+++ b/src/fftw3/kernel/md5.c
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* 
+   independent implementation of Ron Rivest's MD5 message-digest
+   algorithm, based on rfc 1321.
+
+   Optimized for small code size, not speed.  Works as long as
+   sizeof(md5uint) >= 4.
+*/
+
+#include "ifftw.h"
+
+/* sintab[i] = 4294967296.0 * abs(sin((double)(i + 1))) */
+static const md5uint sintab[64] = {
+     0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee,
+     0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501,
+     0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be,
+     0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821,
+     0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa,
+     0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8,
+     0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
+     0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a,
+     0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c,
+     0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70,
+     0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05,
+     0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665,
+     0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039,
+     0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
+     0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1,
+     0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391
+}; 
+
+/* see rfc 1321 section 3.4 */
+static const struct roundtab {
+     char k; 
+     char s;
+} roundtab[64] = {
+     {  0,  7}, {  1, 12}, {  2, 17}, {  3, 22},
+     {  4,  7}, {  5, 12}, {  6, 17}, {  7, 22},
+     {  8,  7}, {  9, 12}, { 10, 17}, { 11, 22},
+     { 12,  7}, { 13, 12}, { 14, 17}, { 15, 22},
+     {  1,  5}, {  6,  9}, { 11, 14}, {  0, 20},
+     {  5,  5}, { 10,  9}, { 15, 14}, {  4, 20},
+     {  9,  5}, { 14,  9}, {  3, 14}, {  8, 20},
+     { 13,  5}, {  2,  9}, {  7, 14}, { 12, 20},
+     {  5,  4}, {  8, 11}, { 11, 16}, { 14, 23},
+     {  1,  4}, {  4, 11}, {  7, 16}, { 10, 23},
+     { 13,  4}, {  0, 11}, {  3, 16}, {  6, 23},
+     {  9,  4}, { 12, 11}, { 15, 16}, {  2, 23},
+     {  0,  6}, {  7, 10}, { 14, 15}, {  5, 21},
+     { 12,  6}, {  3, 10}, { 10, 15}, {  1, 21},
+     {  8,  6}, { 15, 10}, {  6, 15}, { 13, 21},
+     {  4,  6}, { 11, 10}, {  2, 15}, {  9, 21}
+};
+
+#define rol(a, s) ((a << (int)(s)) | (a >> (32 - (int)(s))))
+
+static void doblock(md5sig state, const unsigned char *data)
+{
+     md5uint a, b, c, d, t, x[16];
+     const md5uint msk = 0xffffffffUL;
+     int i;
+
+     /* encode input bytes into md5uint */
+     for (i = 0; i < 16; ++i) {
+	  const unsigned char *p = data + 4 * i;
+	  x[i] = p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24);
+     }
+
+     a = state[0]; b = state[1]; c = state[2]; d = state[3];
+     for (i = 0; i < 64; ++i) {
+	  const struct roundtab *p = roundtab + i;
+	  int round = i / 16;
+	  switch (round) {
+	      case 0: a += (b & c) | (~b & d); break;
+	      case 1: a += (b & d) | (c & ~d); break;
+	      case 2: a += b ^ c ^ d; break;
+	      case 3: a += c ^ (b | ~d); break;
+	  }
+	  a += sintab[i];
+	  a += x[(int)(p->k)];
+	  a &= msk;
+	  t = b + rol(a, p->s);
+	  a = d; d = c; c = b; b = t;
+     }
+     state[0] = (state[0] + a) & msk;
+     state[1] = (state[1] + b) & msk;
+     state[2] = (state[2] + c) & msk;
+     state[3] = (state[3] + d) & msk;
+}
+
+
+void X(md5begin)(md5 *p)
+{
+     p->s[0] = 0x67452301;
+     p->s[1] = 0xefcdab89;
+     p->s[2] = 0x98badcfe;
+     p->s[3] = 0x10325476;
+     p->l = 0;
+}
+
+void X(md5putc)(md5 *p, unsigned char c)
+{
+     p->c[p->l % 64] = c;
+     if (((++p->l) % 64) == 0) doblock(p->s, p->c);
+}
+
+void X(md5end)(md5 *p)
+{
+     unsigned l, i;
+
+     l = 8 * p->l; /* length before padding, in bits */
+
+     /* rfc 1321 section 3.1: padding */
+     X(md5putc)(p, 0x80);
+     while ((p->l % 64) != 56) X(md5putc)(p, 0x00);
+
+     /* rfc 1321 section 3.2: length (little endian) */
+     for (i = 0; i < 8; ++i) {
+	  X(md5putc)(p, l & 0xFF);
+	  l = l >> 8;
+     }
+
+     /* Now p->l % 64 == 0 and signature is in p->s */
+}
diff --git a/src/fftw3/kernel/minmax.c b/src/fftw3/kernel/minmax.c
new file mode 100644
index 0000000..fc6f3a3
--- /dev/null
+++ b/src/fftw3/kernel/minmax.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: minmax.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+int X(imax)(int a, int b)
+{
+     return (a > b) ? a : b;
+}
+
+int X(imin)(int a, int b)
+{
+     return (a < b) ? a : b;
+}
diff --git a/src/fftw3/kernel/ops.c b/src/fftw3/kernel/ops.c
new file mode 100644
index 0000000..9e927c1
--- /dev/null
+++ b/src/fftw3/kernel/ops.c
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: ops.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+void X(ops_zero)(opcnt *dst)
+{
+     dst->add = dst->mul = dst->fma = dst->other = 0;
+}
+
+void X(ops_cpy)(const opcnt *src, opcnt *dst)
+{
+     *dst = *src;
+}
+
+void X(ops_other)(int o, opcnt *dst)
+{
+     X(ops_zero)(dst);
+     dst->other = o;
+}
+
+void X(ops_madd)(int m, const opcnt *a, const opcnt *b, opcnt *dst)
+{
+     dst->add = m * a->add + b->add;
+     dst->mul = m * a->mul + b->mul;
+     dst->fma = m * a->fma + b->fma;
+     dst->other = m * a->other + b->other;
+}
+
+void X(ops_add)(const opcnt *a, const opcnt *b, opcnt *dst)
+{
+     X(ops_madd)(1, a, b, dst);
+}
+
+void X(ops_add2)(const opcnt *a, opcnt *dst)
+{
+     X(ops_add)(a, dst, dst);
+}
+
+void X(ops_madd2)(int m, const opcnt *a, opcnt *dst)
+{
+     X(ops_madd)(m, a, dst, dst);
+}
+
diff --git a/src/fftw3/kernel/pickdim.c b/src/fftw3/kernel/pickdim.c
new file mode 100644
index 0000000..034e6db
--- /dev/null
+++ b/src/fftw3/kernel/pickdim.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "ifftw.h"
+
+/* $Id: pickdim.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* Given a solver which_dim, a vector sz, and whether or not the
+   transform is out-of-place, return the actual dimension index that
+   it corresponds to.  The basic idea here is that we return the
+   which_dim'th valid dimension, starting from the end if
+   which_dim < 0. */
+static int really_pickdim(int which_dim, const tensor *sz, int oop, int *dp)
+{
+     int i;
+     int count_ok = 0;
+     if (which_dim > 0) {
+          for (i = 0; i < sz->rnk; ++i) {
+               if (oop || sz->dims[i].is == sz->dims[i].os)
+                    if (++count_ok == which_dim) {
+                         *dp = i;
+                         return 1;
+                    }
+          }
+     }
+     else if (which_dim < 0) {
+          for (i = sz->rnk; i > 0; --i) {
+               if (oop || sz->dims[i - 1].is == sz->dims[i - 1].os)
+                    if (++count_ok == -which_dim) {
+                         *dp = i - 1;
+                         return 1;
+                    }
+          }
+     }
+     else { /* zero: pick the middle, if valid */
+	  i = sz->rnk / 2 - 1;
+	  if (i < sz->rnk && (oop || sz->dims[i].is == sz->dims[i].os)) {
+	       *dp = i;
+	       return 1;
+	  }
+     }
+     return 0;
+}
+
+/* Like really_pickdim, but only returns 1 if no previous "buddy"
+   which_dim in the buddies list would give the same dim. */
+int X(pickdim)(int which_dim, const int *buddies, int nbuddies,
+	       const tensor *sz, int oop, int *dp)
+{
+     int i, d1;
+
+     if (!really_pickdim(which_dim, sz, oop, dp))
+          return 0;
+
+     /* check whether some buddy solver would produce the same dim.
+        If so, consider this solver unapplicable and let the buddy
+        take care of it.  The smallest-indexed buddy is applicable. */
+     for (i = 0; i < nbuddies; ++i) {
+          if (buddies[i] == which_dim)
+               break;  /* found self */
+          if (really_pickdim(buddies[i], sz, oop, &d1) && *dp == d1)
+               return 0; /* found equivalent buddy */
+     }
+     return 1;
+}
diff --git a/src/fftw3/kernel/planner.c b/src/fftw3/kernel/planner.c
new file mode 100644
index 0000000..1d01cb9
--- /dev/null
+++ b/src/fftw3/kernel/planner.c
@@ -0,0 +1,695 @@
+/*
+ * Copyright (c) 2000 Matteo Frigo
+ * Copyright (c) 2000 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: planner.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+#include "ifftw.h"
+#include <string.h>
+
+/* GNU Coding Standards, Sec. 5.2: "Please write the comments in a GNU
+   program in English, because English is the one language that nearly
+   all programmers in all countries can read."
+
+                    ingemisco tanquam reus
+		    culpa rubet vultus meus
+		    supplicanti parce [rms]
+*/
+
+#define BLESSEDP(solution) ((solution)->flags & BLESSING)
+#define VALIDP(solution) ((solution)->flags & H_VALID)
+#define UNBLESS(flags)  flags &= ~BLESSING
+
+#define MAXNAM 64  /* maximum length of registrar's name.
+		      Used for reading wisdom.  There is no point
+		      in doing this right */
+
+/* Flags f1 subsumes flags f2 iff f1 is less/equally impatient than
+   f2, defining a partial ordering. */
+#define IMPATIENCE(flags) ((flags) & IMPATIENCE_FLAGS)
+#define NONIMPATIENCE(flags) ((flags) & NONIMPATIENCE_FLAGS)
+#define ORDERED(f1, f2) (SUBSUMES(f1, f2) || SUBSUMES(f2, f1))
+#define SUBSUMES(f1, f2) ((IMPATIENCE(f1) & (f2)) == IMPATIENCE(f1))
+
+static unsigned addmod(unsigned a, unsigned b, unsigned p)
+{
+     /* gcc-2.95/sparc produces incorrect code for the fast version below. */
+#if defined(__sparc__) && defined(__GNUC__)
+     /* slow version  */
+     return (a + b) % p;
+#else
+     /* faster version */
+     unsigned c = a + b;
+     return c >= p ? c - p : c;
+#endif
+}
+
+/*
+  slvdesc management:
+*/
+static void sgrow(planner *ego)
+{
+     unsigned osiz = ego->slvdescsiz, nsiz = 1 + osiz + osiz / 4;
+     slvdesc *ntab = (slvdesc *)MALLOC(nsiz * sizeof(slvdesc), SLVDESCS);
+     slvdesc *otab = ego->slvdescs;
+     unsigned i;
+
+     ego->slvdescs = ntab;
+     ego->slvdescsiz = nsiz;
+     for (i = 0; i < osiz; ++i)
+	  ntab[i] = otab[i];
+     X(ifree0)(otab);
+}
+
+static void register_solver(planner *ego, solver *s)
+{
+     slvdesc *n;
+     if (s) { /* add s to solver list */
+	  X(solver_use)(s);
+
+	  if (ego->nslvdesc >= ego->slvdescsiz)
+	       sgrow(ego);
+
+	  n = ego->slvdescs + ego->nslvdesc++;
+
+	  n->slv = s;
+	  n->reg_nam = ego->cur_reg_nam;
+	  n->reg_id = ego->cur_reg_id++;
+	  
+	  A(strlen(n->reg_nam) < MAXNAM);
+	  n->nam_hash = X(hash)(n->reg_nam);
+     }
+}
+
+static int slookup(planner *ego, char *nam, int id)
+{
+     unsigned h = X(hash)(nam); /* used to avoid strcmp in the common case */
+     FORALL_SOLVERS(ego, s, sp, {
+	  UNUSED(s);
+	  if (sp->reg_id == id && sp->nam_hash == h
+	      && !strcmp(sp->reg_nam, nam))
+	       return sp - ego->slvdescs;
+     });
+     return -1;
+}
+
+/*
+  md5-related stuff:
+*/
+
+/* first hash function */
+static unsigned h1(planner *ego, const md5sig s)
+{
+     return s[0] % ego->hashsiz;
+}
+
+/* second hash function (for double hashing) */
+static unsigned h2(planner *ego, const md5sig s)
+{
+     return 1U + s[1] % (ego->hashsiz - 1);
+}
+
+static void md5hash(md5 *m, const problem *p, const planner *plnr)
+{
+     X(md5begin)(m);
+     X(md5unsigned)(m, sizeof(R)); /* so we don't mix different precisions */
+     X(md5unsigned)(m, plnr->problem_flags);
+     X(md5int)(m, plnr->nthr);
+     p->adt->hash(p, m);
+     X(md5end)(m);
+}
+
+static int md5eq(const md5sig a, const md5sig b)
+{
+     return a[0] == b[0] && a[1] == b[1] && a[2] == b[2] && a[3] == b[3];
+}
+
+static void sigcpy(const md5sig a, md5sig b)
+{
+     b[0] = a[0]; b[1] = a[1]; b[2] = a[2]; b[3] = a[3];
+}
+
+/*
+  memoization routines :
+*/
+
+/*
+   liber scriptus proferetur
+   in quo totum continetur
+   unde mundus iudicetur
+*/
+struct solution_s {
+     md5sig s;
+     unsigned short flags;
+     short slvndx;
+};
+
+static solution *hlookup(planner *ego, const md5sig s, unsigned short flags)
+{
+     unsigned g, h = h1(ego, s), d = h2(ego, s);
+
+     ++ego->lookup;
+
+     for (g = h; ; g = addmod(g, d, ego->hashsiz)) {
+	  solution *l = ego->solutions + g;
+	  ++ego->lookup_iter;
+	  if (VALIDP(l)) {
+	       if (md5eq(s, l->s) && ORDERED(l->flags, flags)) { 
+		    ++ego->succ_lookup;
+		    return l; 
+	       }
+	  } else {
+	       return 0;
+	  }
+	  A((g + d) % ego->hashsiz != h);
+     }
+}
+
+
+static void hinsert0(planner *ego, const md5sig s, unsigned short flags,
+		     int slvndx, solution *l)
+{
+     ++ego->insert;
+     if (!l) { 	 
+	  /* search for nonfull slot */
+	  unsigned g, h = h1(ego, s), d = h2(ego, s); 
+	  ++ego->insert_unknown;
+	  for (g = h; ; g = addmod(g, d, ego->hashsiz)) {
+	       ++ego->insert_iter;
+	       l = ego->solutions + g;
+	       if (!VALIDP(l)) break;
+	       A((g + d) % ego->hashsiz != h);
+	  }
+     }
+
+     /* fill slot */
+     l->flags = flags | H_VALID;
+     l->slvndx = (short)slvndx;
+     sigcpy(s, l->s);
+}
+
+static void rehash(planner *ego, unsigned nsiz)
+{
+     unsigned osiz = ego->hashsiz, h;
+     solution *osol = ego->solutions, *nsol;
+
+     nsiz = (unsigned)X(next_prime)((int)nsiz);
+     nsol = (solution *)MALLOC(nsiz * sizeof(solution), HASHT);
+     ++ego->nrehash;
+
+     /* init new table */
+     for (h = 0; h < nsiz; ++h) 
+	  nsol[h].flags = 0;
+
+     /* install new table */
+     ego->hashsiz = nsiz;
+     ego->solutions = nsol;
+
+     /* copy table */
+     for (h = 0; h < osiz; ++h) {
+	  solution *l = osol + h;
+	  if (VALIDP(l))
+	       hinsert0(ego, l->s, l->flags, l->slvndx, 0);
+     }
+
+     X(ifree0)(osol);
+}
+
+static unsigned minsz(unsigned nelem)
+{
+     return 1U + nelem + nelem / 8U;
+}
+
+static unsigned nextsz(unsigned nelem)
+{
+     return minsz(minsz(nelem));
+}
+
+static void hgrow(planner *ego)
+{
+     unsigned nelem = ego->nelem;
+     if (minsz(nelem) >= ego->hashsiz)
+	  rehash(ego, nextsz(nelem));
+}
+
+static void hshrink(planner *ego)
+{
+     unsigned nelem = ego->nelem;
+     /* always rehash after deletions */
+     rehash(ego, nextsz(nelem));
+}
+
+/* inherit blessing, but only if the solver is the same */
+static unsigned short merge_flags(unsigned short dstflags, int dstndx,
+				  unsigned short srcflags, int srcndx)
+{
+     if (srcndx == dstndx)
+	  dstflags |= (srcflags & BLESSING); /* ne me perdas illa die */
+     return dstflags;
+}
+
+static void hinsert(planner *ego, const md5sig s, 
+		    unsigned short flags, int slvndx)
+{
+     solution *l;
+
+     if ((l = hlookup(ego, s, flags))) {
+	  if (SUBSUMES(flags, l->flags)) {
+	       /* overwrite old solution */
+	       flags = merge_flags(flags, slvndx, l->flags, l->slvndx);
+	  } else {
+	       A(SUBSUMES(l->flags, flags));
+	       l->flags = merge_flags(l->flags, l->slvndx, flags, slvndx);
+	       return;
+	  }
+     } else {
+	  ++ego->nelem;
+	  hgrow(ego);
+     }
+     hinsert0(ego, s, flags, slvndx, l);
+}
+
+static void hcurse_subsumed(planner *ego)
+{
+     unsigned h;
+
+     /* unbless any entries that are unreachable because they
+        are subsumed by less-impatient ones.  */
+     for (h = 0; h < ego->hashsiz; ++h) {
+	  solution *l = ego->solutions + h;
+	  if (VALIDP(l)) {
+	       unsigned d = h2(ego, l->s), g = addmod(h, d, ego->hashsiz);
+	       for (; ; g = addmod(g, d, ego->hashsiz)) {
+		    solution *m = ego->solutions + g;
+		    if (VALIDP(m)) {
+			 if (md5eq(l->s, m->s) &&
+			     SUBSUMES(l->flags, m->flags)) {
+			      /* ne cadant in obscurum */
+			      l->flags = merge_flags(l->flags, l->slvndx,
+						     m->flags, m->slvndx);
+
+			      /* cum vix justus sit securus */
+			      UNBLESS(m->flags);
+			 }
+		    }
+		    else break;
+		    A((g + d) % ego->hashsiz != h);
+	       }
+	  }
+     }
+}
+
+
+static void invoke_hook(planner *ego, plan *pln, const problem *p, 
+			int optimalp)
+{
+     if (ego->hook)
+	  ego->hook(pln, p, optimalp);
+}
+
+static void evaluate_plan(planner *ego, plan *pln, const problem *p)
+{
+     if (!BELIEVE_PCOSTP(ego) || pln->pcost == 0.0) {
+	  ego->nplan++;
+
+	  if (ESTIMATEP(ego)) {
+	  estimate:
+	       /* heuristic */
+	       pln->pcost = 0.0
+		    + pln->ops.add
+		    + pln->ops.mul
+		    + 2 * pln->ops.fma
+		    + pln->ops.other;
+	       ego->epcost += pln->pcost;
+	  } else {
+	       double t = X(measure_execution_time)(pln, p);
+
+	       if (t < 0) {  /* unavailable cycle counter */
+		    /* Real programmers can write FORTRAN in any language */
+		    goto estimate;
+	       }
+
+	       pln->pcost = t;
+	       ego->pcost += t;
+	  }
+     }
+     
+     invoke_hook(ego, pln, p, 0);
+}
+
+/* maintain dynamic scoping of flags, nthr: */
+static plan *invoke_solver(planner *ego, problem *p, solver *s, 
+			   unsigned short nflags)
+{
+     unsigned short planner_flags = ego->planner_flags;
+     unsigned problem_flags = ego->problem_flags;
+     int nthr = ego->nthr;
+     plan *pln;
+     ego->planner_flags = nflags;
+     pln = s->adt->mkplan(s, p, ego);
+     ego->problem_flags = problem_flags;
+     ego->nthr = nthr;
+     ego->planner_flags = planner_flags;
+     return pln;
+}
+
+static plan *search(planner *ego, problem *p, slvdesc **descp)
+{
+     plan *best = 0;
+     int best_not_yet_timed = 1;
+     int pass;
+
+     if (NO_SEARCHP(ego)) {
+	  /* D("invalid search for %P %x\n", p, ego->planner_flags); */
+	  return 0;
+     }
+
+     for (pass = 0; pass < 2; ++pass) {
+	  unsigned short nflags = ego->planner_flags;
+	  
+	  if (best) break;
+
+	  switch (pass) {
+	      case 0: 
+		   /* skip pass 0 during exhaustive search */
+		   if (!NO_EXHAUSTIVEP(ego)) continue;
+		   nflags |= NO_UGLY;
+		   break;
+	      case 1:
+		   /* skip pass 1 if NO_UGLY */
+		   if (NO_UGLYP(ego)) continue;
+		   break;
+	  }
+
+          FORALL_SOLVERS(ego, s, sp, {
+	       plan *pln = invoke_solver(ego, p, s, nflags);
+
+	       if (pln) {
+		    if (best) {
+			 if (best_not_yet_timed) {
+			      evaluate_plan(ego, best, p);
+			      best_not_yet_timed = 0;
+			 }
+			 evaluate_plan(ego, pln, p);
+			 if (pln->pcost < best->pcost) {
+			      X(plan_destroy_internal)(best);
+			      best = pln;
+			      *descp = sp;
+			 } else {
+			      X(plan_destroy_internal)(pln);
+			 }
+		    } else {
+			 best = pln;
+			 *descp = sp;
+		    }
+	       }
+	  });
+     }
+
+     return best;
+}
+
+static plan *mkplan(planner *ego, problem *p)
+{
+     plan *pln;
+     md5 m;
+     slvdesc *sp;
+     unsigned short flags;
+     ASSERT_ALIGNED_DOUBLE;
+
+     /* Canonical form. */
+     if (!NO_EXHAUSTIVEP(ego)) ego->planner_flags &= ~NO_UGLY;
+	  
+     ++ego->nprob;
+     md5hash(&m, p, ego);
+
+     pln = 0;
+
+     {
+	  solution *sol; /* new scope for sol */
+
+	  if ((sol = hlookup(ego, m.s, ego->planner_flags))) {
+	       if (SUBSUMES(sol->flags, ego->planner_flags)) {
+		    /* wisdom is acceptable */
+		    if (sol->slvndx < 0) 
+			 return 0;   /* known to be infeasible */
+
+		    /* use solver to obtain a plan */
+		    sp = ego->slvdescs + sol->slvndx;
+		    pln = 
+			 invoke_solver(ego, p, sp->slv, 
+				       (0
+					| NO_SEARCH 
+					| IMPATIENCE(sol->flags)
+					| NONIMPATIENCE(ego->planner_flags) ));
+
+		    /* if (!pln) then the entry is bogus, but
+		       we currently do nothing about it. */
+		    /* CAVEAS: Do not use ``sol'' here, because the
+		       pointer is possibly dangling after the call to
+		       invoke_solver(). */
+	       } else {
+		    A(SUBSUMES(ego->planner_flags, sol->flags));
+	       }
+	  }
+     }
+
+
+     if (!pln)
+	  pln = search(ego, p, &sp);
+
+     flags = ego->planner_flags;
+
+     if (pln) {
+	  /* Postulate de iure that NO_UGLY subsumes ~NO_UGLY if the
+	     problem is feasible. Also postulate that NO_SEARCH
+	     subsumes ~NO_SEARCH. */
+	  flags &= ~(NO_UGLY | NO_SEARCH);
+     }
+
+     hinsert(ego, m.s, flags, pln ? sp - ego->slvdescs : -1);
+
+     if (pln)
+	  invoke_hook(ego, pln, p, 1);
+     return pln;
+}
+
+/* destroy hash table entries.  If FORGET_EVERYTHING, destroy the whole
+   table.  If FORGET_ACCURSED, then destroy entries that are not blessed. */
+static void forget(planner *ego, amnesia a)
+{
+     unsigned h;
+
+     /* garbage-collect while we are at it */ 
+     if (a != FORGET_EVERYTHING)
+	  hcurse_subsumed(ego);
+
+     for (h = 0; h < ego->hashsiz; ++h) {
+	  solution *l = ego->solutions + h;
+	  if (VALIDP(l)) {
+	       if (a == FORGET_EVERYTHING ||
+		   (a == FORGET_ACCURSED && !BLESSEDP(l))) {
+		    /* confutatis maledictis
+		       flammis acribus addictis */
+		    l->flags &= ~H_VALID;
+		    --ego->nelem;
+	       }
+	  }
+     }
+     /* nil inultum remanebit */
+
+     hshrink(ego);
+}
+
+static void htab_destroy(planner *ego)
+{
+     forget(ego, FORGET_EVERYTHING);
+     X(ifree)(ego->solutions);
+     ego->nelem = 0U;
+}
+
+/* FIXME: what sort of version information should we write? */
+#define WISDOM_PREAMBLE PACKAGE "-" VERSION " " STRINGIZE(X(wisdom))
+
+/* tantus labor non sit cassus */
+static void exprt(planner *ego, printer *p)
+{
+     unsigned h;
+
+     hcurse_subsumed(ego);
+
+     p->print(p, "(" WISDOM_PREAMBLE "%(");
+     for (h = 0; h < ego->hashsiz; ++h) {
+	  solution *l = ego->solutions + h;
+	  if (VALIDP(l) && BLESSEDP(l) && l->slvndx >= 0) {
+	       slvdesc *sp = ego->slvdescs + l->slvndx;
+	       /* qui salvandos salvas gratis
+		  salva me fons pietatis */
+	       p->print(p, "(%s %d #x%x #x%M #x%M #x%M #x%M)\n",
+			sp->reg_nam, sp->reg_id, (int)l->flags,
+			l->s[0], l->s[1], l->s[2], l->s[3]);
+	  }
+     }
+     p->print(p, "%))\n");
+}
+
+/* mors stupebit et natura
+   cum resurget creatura */
+static int imprt(planner *ego, scanner *sc)
+{
+     char buf[MAXNAM + 1];
+     md5uint sig[4];
+     int flags;
+     int reg_id;
+     int slvndx;
+     solution *sol;
+
+     if (!sc->scan(sc, "(" WISDOM_PREAMBLE))
+	  return 0; /* don't need to restore hashtable */
+
+     /* make a backup copy of the hash table (cache the hash) */
+     {
+	  unsigned h, hsiz = ego->hashsiz;
+	  sol = (solution *)MALLOC(hsiz * sizeof(solution), HASHT);
+	  for (h = 0; h < hsiz; ++h)
+	       sol[h] = ego->solutions[h];
+     }
+
+     while (1) {
+	  if (sc->scan(sc, ")"))
+	       break;
+
+	  /* qua resurget ex favilla */
+	  if (!sc->scan(sc, "(%*s %d #x%x #x%M #x%M #x%M #x%M)",
+			MAXNAM, buf, &reg_id, &flags, 
+			sig + 0, sig + 1, sig + 2, sig + 3))
+	       goto bad;
+
+	  if ((slvndx = slookup(ego, buf, reg_id)) < 0)
+	       goto bad;
+
+	  /* inter oves locum praesta */
+	  hinsert(ego, sig, (unsigned short)flags, slvndx);
+     }
+
+     X(ifree0)(sol);
+     return 1;
+
+ bad:
+     /* ``The wisdom of FFTW must be above suspicion.'' */
+     X(ifree0)(ego->solutions);
+     ego->solutions = sol;
+     return 0;
+}
+
+/*
+ * create a planner
+ */
+planner *X(mkplanner)(void)
+{
+     static const planner_adt padt = {
+	  register_solver, mkplan, forget, exprt, imprt
+     };
+
+     planner *p = (planner *) MALLOC(sizeof(planner), PLANNERS);
+
+     p->adt = &padt;
+     p->nplan = p->nprob = p->nrehash = 0;
+     p->pcost = p->epcost = 0.0;
+     p->succ_lookup = p->lookup = p->lookup_iter = 0;
+     p->insert = p->insert_iter = p->insert_unknown = 0;
+     p->hook = 0;
+     p->cur_reg_nam = 0;
+
+     p->slvdescs = 0;
+     p->nslvdesc = p->slvdescsiz = 0;
+
+     p->solutions = 0;
+     p->hashsiz = p->nelem = 0U;
+
+     p->problem_flags = 0;
+     p->planner_flags = 0;
+     p->nthr = 1;
+
+     hgrow(p);			/* so that hashsiz > 0 */
+
+     return p;
+}
+
+void X(planner_destroy)(planner *ego)
+{
+     /* destroy hash table */
+     htab_destroy(ego);
+
+     /* destroy solvdesc table */
+     FORALL_SOLVERS(ego, s, sp, {
+	  UNUSED(sp);
+	  X(solver_destroy)(s);
+     });
+
+     X(ifree0)(ego->slvdescs);
+     X(ifree)(ego); /* dona eis requiem */
+}
+
+plan *X(mkplan_d)(planner *ego, problem *p)
+{
+     plan *pln = ego->adt->mkplan(ego, p);
+     X(problem_destroy)(p);
+     return pln;
+}
+
+/*
+ * Debugging code:
+ */
+#ifdef FFTW_DEBUG
+
+void X(planner_dump)(planner *ego, int verbose)
+{
+     unsigned valid = 0, empty = 0, infeasible = 0;
+     unsigned h;
+     UNUSED(verbose); /* historical */
+
+     for (h = 0; h < ego->hashsiz; ++h) {
+	  solution *l = ego->solutions + h; 
+	  if (VALIDP(l)) {
+	       ++valid; 
+	       if (l->slvndx < 0) ++infeasible;
+	  } else
+	       ++empty;
+	  
+     }
+
+     D("nplan = %d\n", ego->nplan);
+     D("nprob = %d\n", ego->nprob);
+     D("pcost = %g\n", ego->pcost);
+     D("epcost = %g\n", ego->epcost);
+     D("lookup = %d\n", ego->lookup);
+     D("succ_lookup = %d\n", ego->succ_lookup);
+     D("lookup_iter = %d\n", ego->lookup_iter);
+     D("insert = %d\n", ego->insert);
+     D("insert_iter = %d\n", ego->insert_iter);
+     D("insert_unknown = %d\n", ego->insert_unknown);
+     D("nrehash = %d\n", ego->nrehash);
+     D("hashsiz = %u\n", ego->hashsiz);
+     D("empty = %d\n", empty);
+     D("valid = %d\n", valid);
+     D("infeasible = %d\n", infeasible);
+     A(ego->nelem == valid);
+}
+
+#endif
diff --git a/src/fftw3/kernel/primes.c b/src/fftw3/kernel/primes.c
new file mode 100644
index 0000000..608e51c
--- /dev/null
+++ b/src/fftw3/kernel/primes.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: primes.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+/***************************************************************************/
+
+/* Rader's algorithm requires lots of modular arithmetic, and if we
+   aren't careful we can have errors due to integer overflows. */
+
+#ifdef SAFE_MULMOD
+
+#  include <limits.h>
+
+/* compute (x * y) mod p, but watch out for integer overflows; we must
+   have x, y >= 0, p > 0.  This routine is slow. */
+int X(safe_mulmod)(int x, int y, int p)
+{
+     if (y == 0 || x <= INT_MAX / y)
+	  return((x * y) % p);
+     else {
+	  int y2 = y/2;
+	  return((X(safe_mulmod)(x, y2, p) +
+		  X(safe_mulmod)(x, y - y2, p)) % p);
+     }
+}
+#endif /* safe_mulmod ('long long' unavailable) */
+
+/***************************************************************************/
+
+/* Compute n^m mod p, where m >= 0 and p > 0.  If we really cared, we
+   could make this tail-recursive. */
+int X(power_mod)(int n, int m, int p)
+{
+     A(p > 0);
+     if (m == 0)
+	  return 1;
+     else if (m % 2 == 0) {
+	  int x = X(power_mod)(n, m / 2, p);
+	  return MULMOD(x, x, p);
+     }
+     else
+	  return MULMOD(n, X(power_mod)(n, m - 1, p), p);
+}
+
+/* the following two routines were contributed by Greg Dionne. */
+static int get_prime_factors(int n, int *primef)
+{
+     int i;
+     int size = 0;
+
+     primef[size++] = 2;
+     do
+	  n >>= 1;
+     while ((n & 1) == 0);
+
+     if (n == 1)
+	  return size;
+
+     for (i = 3; i * i <= n; i += 2)
+	  if (!(n % i)) {
+	       primef[size++] = i;
+	       do
+		    n /= i;
+	       while (!(n % i));
+	  }
+     if (n == 1)
+	  return size;
+     primef[size++] = n;
+     return size;
+}
+
+int X(find_generator)(int p)
+{
+    int n, i, size;
+    int primef[16];     /* smallest number = 32589158477190044730 > 2^64 */
+    int pm1 = p - 1;
+
+    if (p == 2)
+	 return 1;
+
+    size = get_prime_factors(pm1, primef);
+    n = 2;
+    for (i = 0; i < size; i++)
+        if (X(power_mod)(n, pm1 / primef[i], p) == 1) {
+            i = -1;
+            n++;
+        }
+    return n;
+}
+
+/* Return first prime divisor of n  (It would be at best slightly faster to
+   search a static table of primes; there are 6542 primes < 2^16.)  */
+int X(first_divisor)(int n)
+{
+     int i;
+     if (n <= 1)
+	  return n;
+     if (n % 2 == 0)
+	  return 2;
+     for (i = 3; i*i <= n; i += 2)
+	  if (n % i == 0)
+	       return i;
+     return n;
+}
+
+int X(is_prime)(int n)
+{
+     return(n > 1 && X(first_divisor)(n) == n);
+}
+
+int X(next_prime)(int n)
+{
+     while (!X(is_prime)(n)) ++n;
+     return n;
+}
diff --git a/src/fftw3/kernel/print.c b/src/fftw3/kernel/print.c
new file mode 100644
index 0000000..314be7c
--- /dev/null
+++ b/src/fftw3/kernel/print.c
@@ -0,0 +1,210 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: print.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+#include <stddef.h>
+#include <stdarg.h>
+#include <stdio.h>
+
+#define BSZ 64
+
+static void myputs(printer *p, const char *s)
+{
+     char c;
+     while ((c = *s++))
+          p->putchr(p, c);
+}
+
+static void vprint(printer *p, const char *format, va_list ap)
+{
+     char buf[BSZ];
+     const char *s = format;
+     char c;
+     int i;
+
+     for (i = 0; i < p->indent; ++i)
+          p->putchr(p, ' ');
+
+     while ((c = *s++)) {
+          switch (c) {
+	      case '%':
+		   switch ((c = *s++)) {
+		       case 'M': {
+			    /* md5 value */
+			    md5uint x = va_arg(ap, md5uint);
+			    x = 0xffffffffUL & x;
+			    sprintf(buf, "%8.8lx", (unsigned long)x);
+			    goto putbuf;
+		       }
+		       case 'c': {
+			    int x = va_arg(ap, int);
+			    p->putchr(p, x);
+			    break;
+		       }
+		       case 's': {
+			    char *x = va_arg(ap, char *);
+			    if (x)
+				 myputs(p, x);
+			    else
+				 goto putnull;
+			    break;
+		       }
+		       case 'd': {
+			    int x = va_arg(ap, int);
+			    sprintf(buf, "%d", x);
+			    goto putbuf;
+		       }
+		       case 't': {
+			    ptrdiff_t x;
+			    A(*s == 'd');
+			    s += 1;
+			    x = va_arg(ap, ptrdiff_t);
+			    /* should use C99 %td here, but
+			       this is not yet widespread enough */
+			    sprintf(buf, "%ld", (long) x);
+			    goto putbuf;
+		       }
+		       case 'f': case 'e': case 'g': {
+			    char fmt[3] = "%x";
+			    double x = va_arg(ap, double);
+			    fmt[1] = c;
+			    sprintf(buf, fmt, x);
+			    goto putbuf;
+		       }
+		       case 'v': {
+			    /* print optional vector length */
+			    int x = va_arg(ap, int);
+			    if (x > 1) {
+				 sprintf(buf, "-x%d", x);
+				 goto putbuf;
+			    }
+			    break;
+		       }
+		       case 'o': {
+			    /* integer option.  Usage: %oNAME= */
+			    int x = va_arg(ap, int);
+			    if (x)
+				 p->putchr(p, '/');
+			    while ((c = *s++) != '=')
+				 if (x)
+				      p->putchr(p, c);
+			    if (x) {
+				 sprintf(buf, "=%d", x);
+				 goto putbuf;
+			    }
+			    break;
+		       }
+		       case 'u': {
+			    unsigned x = va_arg(ap, unsigned);
+			    sprintf(buf, "%u", x);
+			    goto putbuf;
+		       }
+		       case 'x': {
+			    unsigned x = va_arg(ap, unsigned);
+			    sprintf(buf, "%x", x);
+			    goto putbuf;
+		       }
+		       case '(': {
+			    /* newline, augment indent level */
+			    p->putchr(p, '\n');
+			    p->indent += p->indent_incr;
+			    break;
+		       }
+		       case ')': {
+			    /* decrement indent level */
+			    p->indent -= p->indent_incr;
+			    break;
+		       }
+		       case 'p': {  /* note difference from C's %p */
+			    /* print plan */
+			    plan *x = va_arg(ap, plan *);
+			    if (x) 
+				 x->adt->print(x, p);
+			    else 
+				 goto putnull;
+			    break;
+		       }
+		       case 'P': {
+			    /* print problem */
+			    problem *x = va_arg(ap, problem *);
+			    if (x)
+				 x->adt->print(x, p);
+			    else
+				 goto putnull;
+			    break;
+		       }
+		       case 'T': {
+			    /* print tensor */
+			    tensor *x = va_arg(ap, tensor *);
+			    if (x)
+				 X(tensor_print)(x, p);
+			    else
+				 goto putnull;
+			    break;
+		       }
+		       default:
+			    A(0 /* unknown format */);
+			    break;
+
+		   putbuf:
+			    myputs(p, buf);
+			    break;
+		   putnull:
+			    myputs(p, "(null)");
+			    break;
+		   }
+		   break;
+	      default:
+		   p->putchr(p, c);
+		   break;
+          }
+     }
+}
+
+static void print(printer *p, const char *format, ...)
+{
+     va_list ap;
+     va_start(ap, format);
+     vprint(p, format, ap);
+     va_end(ap);
+}
+
+printer *X(mkprinter)(size_t size, 
+		      void (*putchr)(printer *p, char c),
+		      void (*cleanup)(printer *p))
+{
+     printer *s = (printer *)MALLOC(size, OTHER);
+     s->print = print;
+     s->vprint = vprint;
+     s->putchr = putchr;
+     s->cleanup = cleanup;
+     s->indent = 0;
+     s->indent_incr = 2;
+     return s;
+}
+
+void X(printer_destroy)(printer *p)
+{
+     if (p->cleanup)
+	  p->cleanup(p);
+     X(ifree)(p);
+}
diff --git a/src/fftw3/kernel/scan.c b/src/fftw3/kernel/scan.c
new file mode 100644
index 0000000..f5fa6e1
--- /dev/null
+++ b/src/fftw3/kernel/scan.c
@@ -0,0 +1,204 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: scan.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+#include <string.h>
+#include <stddef.h>
+#include <stdarg.h>
+#include <stdio.h>
+
+#ifdef USE_CTYPE
+#include <ctype.h>
+#else
+/* Screw ctype. On linux, the is* functions call a routine that gets
+   the ctype map in the current locale.  Because this operation is
+   expensive, the map is cached on a per-thread basis.  I am not
+   willing to link this crap with FFTW.  Not over my dead body.
+
+   Sic transit gloria mundi.
+*/
+#undef isspace
+#define isspace(x) ((x) >= 0 && (x) <= ' ')
+#undef isdigit
+#define isdigit(x) ((x) >= '0' && (x) <= '9')
+#undef isupper
+#define isupper(x) ((x) >= 'A' && (x) <= 'Z')
+#undef islower
+#define islower(x) ((x) >= 'a' && (x) <= 'z')
+#endif
+
+static int mygetc(scanner *sc)
+{
+     if (sc->ungotc != EOF) {
+	  int c = sc->ungotc;
+	  sc->ungotc = EOF;
+	  return c;
+     }
+     return(sc->getchr(sc));
+}
+
+#define GETCHR(sc) mygetc(sc)
+
+static void myungetc(scanner *sc, int c)
+{
+     sc->ungotc = c;
+}
+
+#define UNGETCHR(sc, c) myungetc(sc, c)
+
+static void eat_blanks(scanner *sc)
+{
+     int ch;
+     while (ch = GETCHR(sc), isspace(ch))
+          ;
+     UNGETCHR(sc, ch);
+}
+
+static void mygets(scanner *sc, char *s, size_t maxlen)
+{
+     char *s0 = s;
+     int ch;
+
+     A(maxlen > 0);
+     while ((ch = GETCHR(sc)) != EOF && !isspace(ch)
+	    && ch != ')' && ch != '(' && s < s0 + maxlen)
+	  *s++ = ch;
+     *s = 0;
+     UNGETCHR(sc, ch);
+}
+
+static long getlong(scanner *sc, int base, int *ret)
+{
+     int sign = 1, ch, count;
+     long x = 0;     
+
+     ch = GETCHR(sc);
+     if (ch == '-' || ch == '+') {
+	  sign = ch == '-' ? -1 : 1;
+	  ch = GETCHR(sc);
+     }
+     for (count = 0; ; ++count) {
+	  if (isdigit(ch)) 
+	       ch -= '0';
+	  else if (isupper(ch))
+	       ch -= 'A' - 10;
+	  else if (islower(ch))
+	       ch -= 'a' - 10;
+	  else
+	       break;
+	  x = x * base + ch;
+	  ch = GETCHR(sc);
+     }
+     x *= sign;
+     UNGETCHR(sc, ch);
+     *ret = count > 0;
+     return x;
+}
+
+/* vscan is mostly scanf-like, with our additional format specifiers,
+   but with a few twists.  It returns simply 0 or 1 indicating whether
+   the match was successful. '(' and ')' in the format string match
+   those characters preceded by any whitespace.  Finally, if a
+   character match fails, it will ungetchr() the last character back
+   onto the stream. */
+static int vscan(scanner *sc, const char *format, va_list ap)
+{
+     const char *s = format;
+     char c;
+     int ch = 0;
+     size_t fmt_len;
+
+     while ((c = *s++)) {
+	  fmt_len = 0;
+          switch (c) {
+	      case '%':
+	  getformat:
+		   switch ((c = *s++)) {
+		       case 's': {
+			    char *x = va_arg(ap, char *);
+			    mygets(sc, x, fmt_len);
+			    break;
+		       }
+		       case 'd': {
+			    int *x = va_arg(ap, int *);
+			    *x = (int) getlong(sc, 10, &ch);
+			    if (!ch) return 0;
+			    break;
+		       }
+		       case 'x': {
+			    int *x = va_arg(ap, int *);
+			    *x = (int) getlong(sc, 16, &ch);
+			    if (!ch) return 0;
+			    break;
+		       }
+		       case 'M': {
+			    md5uint *x = va_arg(ap, md5uint *);
+			    *x = 0xffffffffUL & getlong(sc, 16, &ch);
+			    if (!ch) return 0;
+			    break;
+		       }
+		       case '*': {
+			    if ((fmt_len = va_arg(ap, int)) <= 0) return 0;
+			    goto getformat;
+		       }
+		       default:
+			    A(0 /* unknown format */);
+			    break;
+		   }
+		   break;
+	      default:
+		   if (isspace(c) || c == '(' || c == ')')
+			eat_blanks(sc);
+		   if (!isspace(c) && (ch = GETCHR(sc)) != c) {
+			UNGETCHR(sc, ch);
+			return 0;
+		   }
+		   break;
+          }
+     }
+     return 1;
+}
+
+static int scan(scanner *sc, const char *format, ...)
+{
+     int ret;
+     va_list ap;
+     va_start(ap, format);
+     ret = vscan(sc, format, ap);
+     va_end(ap);
+     return ret;
+}
+
+scanner *X(mkscanner)(size_t size, int (*getchr)(scanner *sc))
+{
+     scanner *s = (scanner *)MALLOC(size, OTHER);
+     s->scan = scan;
+     s->vscan = vscan;
+     s->getchr = getchr;
+     s->ungotc = EOF;
+     return s;
+}
+
+void X(scanner_destroy)(scanner *sc)
+{
+     X(ifree)(sc);
+}
diff --git a/src/fftw3/kernel/solver.c b/src/fftw3/kernel/solver.c
new file mode 100644
index 0000000..4bfb899
--- /dev/null
+++ b/src/fftw3/kernel/solver.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: solver.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+solver *X(mksolver)(size_t size, const solver_adt *adt)
+{
+     solver *s = (solver *)MALLOC(size, SOLVERS);
+
+     s->adt = adt;
+     s->refcnt = 0;
+     return s;
+}
+
+void X(solver_use)(solver *ego)
+{
+     ++ego->refcnt;
+}
+
+void X(solver_destroy)(solver *ego)
+{
+     if ((--ego->refcnt) == 0)
+          X(ifree)(ego);
+}
+
+void X(solver_register)(planner *plnr, solver *s)
+{
+     plnr->adt->register_solver(plnr, s);
+}
diff --git a/src/fftw3/kernel/solvtab.c b/src/fftw3/kernel/solvtab.c
new file mode 100644
index 0000000..496915e
--- /dev/null
+++ b/src/fftw3/kernel/solvtab.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: solvtab.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+void X(solvtab_exec)(solvtab tbl, planner *p)
+{
+     for (; tbl->reg_nam; ++tbl) {
+	  p->cur_reg_nam = tbl->reg_nam;
+	  p->cur_reg_id = 0;
+	  tbl->reg(p);
+     }
+     p->cur_reg_nam = 0;
+}
diff --git a/src/fftw3/kernel/square.c b/src/fftw3/kernel/square.c
new file mode 100644
index 0000000..4b5afab
--- /dev/null
+++ b/src/fftw3/kernel/square.c
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: square.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+int X(square)(int x)
+{
+     return x * x;
+}
diff --git a/src/fftw3/kernel/stride.c b/src/fftw3/kernel/stride.c
new file mode 100644
index 0000000..fda049f
--- /dev/null
+++ b/src/fftw3/kernel/stride.c
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: stride.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+#include "ifftw.h"
+
+#ifdef PRECOMPUTE_ARRAY_INDICES
+stride X(mkstride)(int n, int s)
+{
+     int i;
+     int *p = (int *) MALLOC(n * sizeof(int), STRIDES);
+
+     for (i = 0; i < n; ++i)
+          p[i] = s * i;
+
+     return p;
+}
+
+void X(stride_destroy)(stride p)
+{
+     X(ifree0)(p);
+}
+
+#endif
diff --git a/src/fftw3/kernel/tensor.c b/src/fftw3/kernel/tensor.c
new file mode 100644
index 0000000..1161963
--- /dev/null
+++ b/src/fftw3/kernel/tensor.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: tensor.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+tensor *X(mktensor)(int rnk) 
+{
+     tensor *x;
+
+     A(rnk >= 0);
+
+#if defined(STRUCT_HACK_KR)
+     if (FINITE_RNK(rnk) && rnk > 1)
+	  x = (tensor *)MALLOC(sizeof(tensor) + (rnk - 1) * sizeof(iodim),
+				    TENSORS);
+     else
+	  x = (tensor *)MALLOC(sizeof(tensor), TENSORS);
+#elif defined(STRUCT_HACK_C99)
+     if (FINITE_RNK(rnk))
+	  x = (tensor *)MALLOC(sizeof(tensor) + rnk * sizeof(iodim),
+				    TENSORS);
+     else
+	  x = (tensor *)MALLOC(sizeof(tensor), TENSORS);
+#else
+     x = (tensor *)MALLOC(sizeof(tensor), TENSORS);
+     if (FINITE_RNK(rnk) && rnk > 0)
+          x->dims = (iodim *)MALLOC(sizeof(iodim) * rnk, TENSORS);
+     else
+          x->dims = 0;
+#endif
+
+     x->rnk = rnk;
+     return x;
+}
+
+void X(tensor_destroy)(tensor *sz)
+{
+#if !defined(STRUCT_HACK_C99) && !defined(STRUCT_HACK_KR)
+     X(ifree0)(sz->dims);
+#endif
+     X(ifree)(sz);
+}
+
+int X(tensor_sz)(const tensor *sz)
+{
+     int i, n = 1;
+
+     if (!FINITE_RNK(sz->rnk))
+          return 0;
+
+     for (i = 0; i < sz->rnk; ++i)
+          n *= sz->dims[i].n;
+     return n;
+}
+
+void X(tensor_md5)(md5 *p, const tensor *t)
+{
+     int i;
+     X(md5int)(p, t->rnk);
+     if (FINITE_RNK(t->rnk)) {
+	  for (i = 0; i < t->rnk; ++i) {
+	       const iodim *q = t->dims + i;
+	       X(md5int)(p, q->n);
+	       X(md5int)(p, q->is);
+	       X(md5int)(p, q->os);
+	  }
+     }
+}
+
+/* treat a (rank <= 1)-tensor as a rank-1 tensor, extracting
+   appropriate n, is, and os components */
+int X(tensor_tornk1)(const tensor *t, int *n, int *is, int *os)
+{
+     A(t->rnk <= 1);
+     if (t->rnk == 1) {
+	  const iodim *vd = t->dims;
+          *n = vd[0].n;
+          *is = vd[0].is;
+          *os = vd[0].os;
+     } else {
+          *n = 1;
+          *is = *os = 0;
+     }
+     return 1;
+}
+
+void X(tensor_print)(const tensor *x, printer *p)
+{
+     if (FINITE_RNK(x->rnk)) {
+	  int i;
+	  int first = 1;
+	  p->print(p, "(");
+	  for (i = 0; i < x->rnk; ++i) {
+	       const iodim *d = x->dims + i;
+	       p->print(p, "%s(%d %d %d)", 
+			first ? "" : " ",
+			d->n, d->is, d->os);
+	       first = 0;
+	  }
+	  p->print(p, ")");
+     } else {
+	  p->print(p, "rank-minfty"); 
+     }
+}
diff --git a/src/fftw3/kernel/tensor1.c b/src/fftw3/kernel/tensor1.c
new file mode 100644
index 0000000..06ad4dc
--- /dev/null
+++ b/src/fftw3/kernel/tensor1.c
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: tensor1.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+tensor *X(mktensor_0d)(void)
+{
+     return X(mktensor(0));
+}
+
+tensor *X(mktensor_1d)(int n, int is, int os)
+{
+     tensor *x = X(mktensor)(1);
+     x->dims[0].n = n;
+     x->dims[0].is = is;
+     x->dims[0].os = os;
+     return x;
+}
diff --git a/src/fftw3/kernel/tensor2.c b/src/fftw3/kernel/tensor2.c
new file mode 100644
index 0000000..a45e164
--- /dev/null
+++ b/src/fftw3/kernel/tensor2.c
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: tensor2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+tensor *X(mktensor_2d)(int n0, int is0, int os0,
+                      int n1, int is1, int os1)
+{
+     tensor *x = X(mktensor)(2);
+     x->dims[0].n = n0;
+     x->dims[0].is = is0;
+     x->dims[0].os = os0;
+     x->dims[1].n = n1;
+     x->dims[1].is = is1;
+     x->dims[1].os = os1;
+     return x;
+}
+
diff --git a/src/fftw3/kernel/tensor4.c b/src/fftw3/kernel/tensor4.c
new file mode 100644
index 0000000..9b6cd28
--- /dev/null
+++ b/src/fftw3/kernel/tensor4.c
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: tensor4.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+int X(tensor_max_index)(const tensor *sz)
+{
+     int i;
+     int n = 0;
+
+     A(FINITE_RNK(sz->rnk));
+     for (i = 0; i < sz->rnk; ++i) {
+          const iodim *p = sz->dims + i;
+          n += (p->n - 1) * X(imax)(X(iabs)(p->is), X(iabs)(p->os));
+     }
+     return n;
+}
+
+#define tensor_min_xstride(sz, xs) {			\
+     A(FINITE_RNK(sz->rnk));				\
+     if (sz->rnk == 0) return 0;			\
+     else {						\
+          int i;					\
+          int s = X(iabs)(sz->dims[0].xs);		\
+          for (i = 1; i < sz->rnk; ++i)			\
+               s = X(imin)(s, X(iabs)(sz->dims[i].xs));	\
+          return s;					\
+     }							\
+}
+
+int X(tensor_min_istride)(const tensor *sz) tensor_min_xstride(sz, is)
+int X(tensor_min_ostride)(const tensor *sz) tensor_min_xstride(sz, os)
+
+int X(tensor_min_stride)(const tensor *sz)
+{
+     return X(imin)(X(tensor_min_istride)(sz), X(tensor_min_ostride)(sz));
+}
+
+int X(tensor_inplace_strides)(const tensor *sz)
+{
+     int i;
+     A(FINITE_RNK(sz->rnk));
+     for (i = 0; i < sz->rnk; ++i) {
+          const iodim *p = sz->dims + i;
+          if (p->is != p->os)
+               return 0;
+     }
+     return 1;
+}
+
+int X(tensor_inplace_strides2)(const tensor *a, const tensor *b)
+{
+     return X(tensor_inplace_strides(a)) && X(tensor_inplace_strides(b));
+}
diff --git a/src/fftw3/kernel/tensor5.c b/src/fftw3/kernel/tensor5.c
new file mode 100644
index 0000000..8144930
--- /dev/null
+++ b/src/fftw3/kernel/tensor5.c
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: tensor5.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+static void dimcpy(iodim *dst, const iodim *src, int rnk)
+{
+     int i;
+     if (FINITE_RNK(rnk))
+          for (i = 0; i < rnk; ++i)
+               dst[i] = src[i];
+}
+
+tensor *X(tensor_copy)(const tensor *sz)
+{
+     tensor *x = X(mktensor)(sz->rnk);
+     dimcpy(x->dims, sz->dims, sz->rnk);
+     return x;
+}
+
+/* like X(tensor_copy), but makes strides in-place by
+   setting os = is if k == INPLACE_IS or is = os if k == INPLACE_OS. */
+tensor *X(tensor_copy_inplace)(const tensor *sz, inplace_kind k)
+{
+     tensor *x = X(tensor_copy)(sz);
+     if (FINITE_RNK(x->rnk)) {
+	  int i;
+	  if (k == INPLACE_OS)
+	       for (i = 0; i < x->rnk; ++i)
+		    x->dims[i].is = x->dims[i].os;
+	  else
+	       for (i = 0; i < x->rnk; ++i)
+		    x->dims[i].os = x->dims[i].is;
+     }
+     return x;
+}
+
+/* Like X(tensor_copy), but copy all of the dimensions *except*
+   except_dim. */
+tensor *X(tensor_copy_except)(const tensor *sz, int except_dim)
+{
+     tensor *x;
+
+     A(FINITE_RNK(sz->rnk) && sz->rnk >= 1 && except_dim < sz->rnk);
+     x = X(mktensor)(sz->rnk - 1);
+     dimcpy(x->dims, sz->dims, except_dim);
+     dimcpy(x->dims + except_dim, sz->dims + except_dim + 1,
+            x->rnk - except_dim);
+     return x;
+}
+
+/* Like X(tensor_copy), but copy only rnk dimensions starting
+   with start_dim. */
+tensor *X(tensor_copy_sub)(const tensor *sz, int start_dim, int rnk)
+{
+     tensor *x;
+
+     A(FINITE_RNK(sz->rnk) && start_dim + rnk <= sz->rnk);
+     x = X(mktensor)(rnk);
+     dimcpy(x->dims, sz->dims + start_dim, rnk);
+     return x;
+}
+
+tensor *X(tensor_append)(const tensor *a, const tensor *b)
+{
+     if (!FINITE_RNK(a->rnk) || !FINITE_RNK(b->rnk)) {
+          return X(mktensor)(RNK_MINFTY);
+     } else {
+	  tensor *x = X(mktensor)(a->rnk + b->rnk);
+          dimcpy(x->dims, a->dims, a->rnk);
+          dimcpy(x->dims + a->rnk, b->dims, b->rnk);
+	  return x;
+     }
+}
diff --git a/src/fftw3/kernel/tensor7.c b/src/fftw3/kernel/tensor7.c
new file mode 100644
index 0000000..ceae2c4
--- /dev/null
+++ b/src/fftw3/kernel/tensor7.c
@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: tensor7.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+/* total order among iodim's */
+int X(dimcmp)(const iodim *a, const iodim *b)
+{
+     if (b->is != a->is)
+          return (b->is - a->is);	/* shorter strides go later */
+     if (b->os != a->os)
+          return (b->os - a->os);	/* shorter strides go later */
+     return (int)(a->n - b->n);	        /* larger n's go later */
+}
+
+/* Like tensor_copy, but eliminate n == 1 dimensions, which
+   never affect any transform or transform vector.
+ 
+   Also, we sort the tensor into a canonical order of decreasing
+   is.  In general, processing a loop/array in order of
+   decreasing stride will improve locality; sorting also makes the
+   analysis in fftw_tensor_contiguous (below) easier.  The choice
+   of is over os is mostly arbitrary, and hopefully
+   shouldn't affect things much.  Normally, either the os will be
+   in the same order as is (for e.g. multi-dimensional
+   transforms) or will be in opposite order (e.g. for Cooley-Tukey
+   recursion).  (Both forward and backwards traversal of the tensor
+   are considered e.g. by vrank-geq1, so sorting in increasing
+   vs. decreasing order is not really important.) */
+tensor *X(tensor_compress)(const tensor *sz)
+{
+     int i, rnk;
+     tensor *x;
+
+     A(FINITE_RNK(sz->rnk));
+     for (i = rnk = 0; i < sz->rnk; ++i) {
+          A(sz->dims[i].n > 0);
+          if (sz->dims[i].n != 1)
+               ++rnk;
+     }
+
+     x = X(mktensor)(rnk);
+     for (i = rnk = 0; i < sz->rnk; ++i) {
+          if (sz->dims[i].n != 1)
+               x->dims[rnk++] = sz->dims[i];
+     }
+
+     if (rnk) {
+	  /* God knows how qsort() behaves if n==0 */
+	  qsort(x->dims, (size_t)x->rnk, sizeof(iodim),
+		(int (*)(const void *, const void *))X(dimcmp));
+     }
+
+     return x;
+}
+
+/* Return whether the strides of a and b are such that they form an
+   effective contiguous 1d array.  Assumes that a.is >= b.is. */
+static int strides_contig(iodim *a, iodim *b)
+{
+     return (a->is == b->is * (int)b->n &&
+             a->os == b->os * (int)b->n);
+}
+
+/* Like tensor_compress, but also compress into one dimension any
+   group of dimensions that form a contiguous block of indices with
+   some stride.  (This can safely be done for transform vector sizes.) */
+tensor *X(tensor_compress_contiguous)(const tensor *sz)
+{
+     int i, rnk;
+     tensor *sz2, *x;
+
+     if (X(tensor_sz)(sz) == 0) 
+	  return X(mktensor)(RNK_MINFTY);
+
+     sz2 = X(tensor_compress)(sz);
+     A(FINITE_RNK(sz2->rnk));
+
+     if (sz2->rnk < 2)		/* nothing to compress */
+          return sz2;
+
+     for (i = rnk = 1; i < sz2->rnk; ++i)
+          if (!strides_contig(sz2->dims + i - 1, sz2->dims + i))
+               ++rnk;
+
+     x = X(mktensor)(rnk);
+     x->dims[0] = sz2->dims[0];
+     for (i = rnk = 1; i < sz2->rnk; ++i) {
+          if (strides_contig(sz2->dims + i - 1, sz2->dims + i)) {
+               x->dims[rnk - 1].n *= sz2->dims[i].n;
+               x->dims[rnk - 1].is = sz2->dims[i].is;
+               x->dims[rnk - 1].os = sz2->dims[i].os;
+          } else {
+               A(rnk < x->rnk);
+               x->dims[rnk++] = sz2->dims[i];
+          }
+     }
+
+     X(tensor_destroy)(sz2);
+     return x;
+}
+
+/* The inverse of X(tensor_append): splits the sz tensor into
+   tensor a followed by tensor b, where a's rank is arnk. */
+void X(tensor_split)(const tensor *sz, tensor **a, int arnk, tensor **b)
+{
+     A(FINITE_RNK(sz->rnk) && FINITE_RNK(arnk));
+
+     *a = X(tensor_copy_sub)(sz, 0, arnk);
+     *b = X(tensor_copy_sub)(sz, arnk, sz->rnk - arnk);
+}
diff --git a/src/fftw3/kernel/tensor8.c b/src/fftw3/kernel/tensor8.c
new file mode 100644
index 0000000..05e9b47
--- /dev/null
+++ b/src/fftw3/kernel/tensor8.c
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: tensor8.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+void X(tensor_destroy2)(tensor *a, tensor *b)
+{
+     X(tensor_destroy)(a);
+     X(tensor_destroy)(b);
+}
+
+void X(tensor_destroy4)(tensor *a, tensor *b, tensor *c, tensor *d)
+{
+     X(tensor_destroy2)(a, b);
+     X(tensor_destroy2)(c, d);
+}
diff --git a/src/fftw3/kernel/tensor9.c b/src/fftw3/kernel/tensor9.c
new file mode 100644
index 0000000..33ddf45
--- /dev/null
+++ b/src/fftw3/kernel/tensor9.c
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: tensor9.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+int X(tensor_kosherp)(const tensor *x)
+{
+     int i;
+
+     if (x->rnk < 0) return 0;
+
+     if (FINITE_RNK(x->rnk)) {
+	  for (i = 0; i < x->rnk; ++i)
+	       if (x->dims[i].n < 0)
+		    return 0;
+     }
+     return 1;
+}
diff --git a/src/fftw3/kernel/timer.c b/src/fftw3/kernel/timer.c
new file mode 100644
index 0000000..72969ad
--- /dev/null
+++ b/src/fftw3/kernel/timer.c
@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: timer.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "ifftw.h"
+
+#ifdef HAVE_UNISTD_H
+#  include <unistd.h>
+#endif
+
+#ifndef WITH_SLOW_TIMER
+#  include "cycle.h"
+#else
+#  if TIME_WITH_SYS_TIME
+#   include <sys/time.h>
+#   include <time.h>
+#  else
+#   if HAVE_SYS_TIME_H
+#    include <sys/time.h>
+#   else
+#    include <time.h>
+#   endif
+#  endif
+#endif
+
+#ifndef FFTW_TIME_LIMIT
+#define FFTW_TIME_LIMIT 2.0  /* don't run for more than two seconds */
+#endif
+
+#ifdef HAVE_BSDGETTIMEOFDAY
+#ifndef HAVE_GETTIMEOFDAY
+#define gettimeofday BSDgettimeofday
+#define HAVE_GETTIMEOFDAY 1
+#endif
+#endif
+
+#if defined(HAVE_GETTIMEOFDAY) && !defined(HAVE_SECONDS_TIMER)
+typedef struct timeval seconds;
+
+static seconds getseconds(void)
+{
+     struct timeval tv;
+     gettimeofday(&tv, 0);
+     return tv;
+}
+
+static double elapsed_sec(seconds t1, seconds t0)
+{
+     return (double)(t1.tv_sec - t0.tv_sec) +
+	  (double)(t1.tv_usec - t0.tv_usec) * 1.0E-6;
+}
+
+#  define TIME_MIN_SEC 1.0e-2 /* from fftw2 */
+#  define HAVE_SECONDS_TIMER
+#endif
+
+#ifndef HAVE_SECONDS_TIMER
+#  include <time.h>
+
+typedef clock_t seconds;
+
+static seconds getseconds(void) { return clock(); }
+
+static double elapsed_sec(seconds t1, seconds t0)
+{
+     return ((double) (t1 - t0)) / CLOCKS_PER_SEC;
+}
+
+#  define TIME_MIN_SEC 2.0e-1 /* from fftw2 */
+#  define HAVE_SECONDS_TIMER
+#endif
+
+#ifdef WITH_SLOW_TIMER
+/* excruciatingly slow; only use this if there is no choice! */
+typedef seconds ticks;
+#  define getticks getseconds
+#  define elapsed elapsed_sec
+#  define TIME_MIN TIME_MIN_SEC
+#  define TIME_REPEAT 4 /* from fftw2 */
+#  define HAVE_TICK_COUNTER
+#endif
+
+#ifdef HAVE_TICK_COUNTER
+
+#  ifndef TIME_MIN
+#    define TIME_MIN 100.0
+#  endif
+
+#  ifndef TIME_REPEAT
+#    define TIME_REPEAT 8
+#  endif
+
+  static double measure(plan *pln, const problem *p, int iter)
+  {
+       ticks t0, t1;
+       int i;
+
+       t0 = getticks();
+       for (i = 0; i < iter; ++i) 
+	    pln->adt->solve(pln, p);
+       t1 = getticks();
+       return elapsed(t1, t0);
+  }
+
+
+  double X(measure_execution_time)(plan *pln, const problem *p)
+  {
+       seconds begin, now;
+       double t, tmax, tmin;
+       int iter;
+       int repeat;
+
+       AWAKE(pln, 1);
+       p->adt->zero(p);
+
+  start_over:
+       for (iter = 1; iter; iter *= 2) {
+	    tmin = 1.0E10;
+	    tmax = -1.0E10;
+
+	    begin = getseconds();
+	    /* repeat the measurement TIME_REPEAT times */
+	    for (repeat = 0; repeat < TIME_REPEAT; ++repeat) {
+		 t = measure(pln, p, iter);
+
+		 if (t < 0)
+		      goto start_over;
+
+		 if (t < tmin)
+		      tmin = t;
+		 if (t > tmax)
+		      tmax = t;
+
+		 /* do not run for too long */
+		 now = getseconds();
+		 t = elapsed_sec(now, begin);
+
+		 if (t > FFTW_TIME_LIMIT)
+		      break;
+	    }
+
+	    if (tmin >= TIME_MIN) {
+		 tmin /= (double) iter;
+		 tmax /= (double) iter;
+		 AWAKE(pln, 0);
+		 return tmin;
+	    }
+       }
+       goto start_over; /* may happen if timer is screwed up */
+  }
+
+#else /* no cycle counter */
+
+  double X(measure_execution_time)(plan *pln, const problem *p)
+  {
+       UNUSED(p);
+       UNUSED(pln);
+       return -1.0;
+  }
+
+#endif
diff --git a/src/fftw3/kernel/transpose.c b/src/fftw3/kernel/transpose.c
new file mode 100644
index 0000000..fb489bd
--- /dev/null
+++ b/src/fftw3/kernel/transpose.c
@@ -0,0 +1,430 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* transposes of unit-stride arrays, including arrays of N-tuples and
+   non-square matrices, using cache-oblivious recursive algorithms */
+
+#include "ifftw.h"
+#include <string.h> /* memcpy */
+
+#define CUTOFF 8 /* size below which we do a naive transpose */
+
+/*************************************************************************/
+/* some utilities for the solvers */
+
+static int Ntuple_transposable(const iodim *a, const iodim *b,
+			       int vl, int s, R *ri, R *ii)
+{
+     return(2 == s && (ii == ri + 1 || ri == ii + 1)
+	    &&
+	    ((a->is == b->os && a->is == (vl*2)
+	      && a->os == b->n * (vl*2) && b->is == a->n * (vl*2))
+	     ||
+	     (a->os == b->is && a->os == (vl*2)
+	      && a->is == b->n * (vl*2) && b->os == a->n * (vl*2))));
+}
+
+
+/* our solvers' transpose routines work for square matrices of arbitrary
+   stride, or for non-square matrices of a given vl*vl2 corresponding
+   to the N of the Ntuple with vl2 == s. */
+int X(transposable)(const iodim *a, const iodim *b,
+		    int vl, int s, R *ri, R *ii)
+{
+     return ((a->n == b->n && a->os == b->is && a->is == b->os)
+	     || Ntuple_transposable(a, b, vl, s, ri, ii));
+}
+
+static int gcd(int a, int b)
+{
+     int r;
+     do {
+	  r = a % b;
+	  a = b;
+	  b = r;
+     } while (r != 0);
+     
+     return a;
+}
+
+/* all of the solvers need to extract n, m, d, n/d, and m/d from the
+   two iodims, so we put it here to save code space */
+void X(transpose_dims)(const iodim *a, const iodim *b,
+		       int *n, int *m, int *d, int *nd, int *md)
+{
+     int n0, m0, d0;
+     /* matrix should be n x m, row-major */
+     if (a->is < b->is) {
+	  *n = n0 = b->n;
+	  *m = m0 = a->n;
+     }
+     else {
+	  *n = n0 = a->n;
+	  *m = m0 = b->n;
+     }
+     *d = d0 = gcd(n0, m0);
+     *nd = n0 / d0;
+     *md = m0 / d0;
+}
+
+/* use the simple square transpose in the solver for square matrices
+   that aren't too big or which have the wrong stride */
+int X(transpose_simplep)(const iodim *a, const iodim *b, int vl, int s,
+			 R *ri, R *ii)
+{
+     return (a->n == b->n &&
+	     (a->n*(vl*2) < CUTOFF 
+	      ||  !Ntuple_transposable(a, b, vl, s, ri, ii)));
+}
+
+/* use the slow general transpose if the buffer would be more than 1/8
+   the whole transpose and the transpose is fairly big.
+   (FIXME: use the CONSERVE_MEMORY flag?) */
+int X(transpose_slowp)(const iodim *a, const iodim *b, int N)
+{
+     int d = gcd(a->n, b->n);
+     return (d < 8 && (a->n * b->n * N) / d > 65536);
+}
+
+/*************************************************************************/
+/* Out-of-place transposes: */
+
+/* Transpose A (n x m) to B (m x n), where A and B are stored
+   as n x fda and m x fda arrays, respectively, operating on N-tuples: */
+static void rec_transpose_Ntuple(R *A, R *B, int n, int m, int fda, int fdb,
+			  int N)
+{
+     if (n == 1 || m == 1 || (n + m) * N < CUTOFF*2) {
+	  int i, j, k;
+	  for (i = 0; i < n; ++i) {
+	       for (j = 0; j < m; ++j) {
+		    for (k = 0; k < N; ++k) { /* FIXME: unroll */
+			 B[(j*fdb + i) * N + k] = A[(i*fda + j) * N + k];
+		    }
+	       }
+	  }
+     }
+     else if (n > m) {
+	  int n2 = n / 2;
+	  rec_transpose_Ntuple(A, B, n2, m, fda, fdb, N);
+	  rec_transpose_Ntuple(A + n2*N*fda, B + n2*N, n - n2, m, fda, fdb, N);
+     }
+     else {
+	  int m2 = m / 2;
+	  rec_transpose_Ntuple(A, B, n, m2, fda, fdb, N);
+	  rec_transpose_Ntuple(A + m2*N, B + m2*N*fdb, n, m - m2, fda, fdb, N);
+     }
+}
+
+/*************************************************************************/
+/* In-place transposes of square matrices of N-tuples: */
+
+/* Transpose both A and B, where A is n x m and B is m x n, storing
+   the transpose of A in B and the transpose of B in A.  A and B
+   are actually stored as n x fda and m x fda arrays. */
+static void rec_transpose_swap_Ntuple(R *A, R *B, int n, int m, int fda, int N)
+{
+     if (n == 1 || m == 1 || (n + m) * N <= CUTOFF*2) {
+	  switch (N) {
+	      case 1: {
+		   int i, j;
+		   for (i = 0; i < n; ++i) {
+			for (j = 0; j < m; ++j) {
+			     R a = A[(i*fda + j)];
+			     A[(i*fda + j)] = B[(j*fda + i)];
+			     B[(j*fda + i)] = a;
+			}
+		   }
+		   break;
+	      }
+	      case 2: {
+		   int i, j;
+		   for (i = 0; i < n; ++i) {
+			for (j = 0; j < m; ++j) {
+			     R a0 = A[(i*fda + j) * 2 + 0];
+			     R a1 = A[(i*fda + j) * 2 + 1];
+			     A[(i*fda + j) * 2 + 0] = B[(j*fda + i) * 2 + 0];
+			     A[(i*fda + j) * 2 + 1] = B[(j*fda + i) * 2 + 1];
+			     B[(j*fda + i) * 2 + 0] = a0;
+			     B[(j*fda + i) * 2 + 1] = a1;
+			}
+		   }
+		   break;
+	      }
+	      default: {
+		   int i, j, k;
+		   for (i = 0; i < n; ++i) {
+			for (j = 0; j < m; ++j) {
+			     for (k = 0; k < N; ++k) {
+				  R a = A[(i*fda + j) * N + k];
+				  A[(i*fda + j) * N + k] = 
+				       B[(j*fda + i) * N + k];
+				  B[(j*fda + i) * N + k] = a;
+			     }
+			}
+		   }
+	      }
+	  }
+     } else if (n > m) {
+	  int n2 = n / 2;
+	  rec_transpose_swap_Ntuple(A, B, n2, m, fda, N);
+	  rec_transpose_swap_Ntuple(A + n2*N*fda, B + n2*N, n - n2, m, fda, N);
+     }
+     else {
+	  int m2 = m / 2;
+	  rec_transpose_swap_Ntuple(A, B, n, m2, fda, N);
+	  rec_transpose_swap_Ntuple(A + m2*N, B + m2*N*fda, n, m - m2, fda, N);
+     }
+}
+
+/* Transpose A, an n x n matrix (stored as n x fda), in-place. */
+static void rec_transpose_sq_ip_Ntuple(R *A, int n, int fda, int N)
+{
+     if (n == 1)
+	  return;
+     else if (n*N <= CUTOFF) {
+	  switch (N) {
+	      case 1: {
+		   int i, j;
+		   for (i = 0; i < n; ++i) {
+			for (j = i + 1; j < n; ++j) {
+			     R a = A[(i*fda + j)];
+			     A[(i*fda + j)] = A[(j*fda + i)];
+			     A[(j*fda + i)] = a;
+			}
+		   }
+		   break;
+	      }
+	      case 2: {
+		   int i, j;
+		   for (i = 0; i < n; ++i) {
+			for (j = i + 1; j < n; ++j) {
+			     R a0 = A[(i*fda + j) * 2 + 0];
+			     R a1 = A[(i*fda + j) * 2 + 1];
+			     A[(i*fda + j) * 2 + 0] = A[(j*fda + i) * 2 + 0];
+			     A[(i*fda + j) * 2 + 1] = A[(j*fda + i) * 2 + 1];
+			     A[(j*fda + i) * 2 + 0] = a0;
+			     A[(j*fda + i) * 2 + 1] = a1;
+			}
+		   }
+		   break;
+	      }
+	      default: {
+		   int i, j, k;
+		   for (i = 0; i < n; ++i) {
+			for (j = i + 1; j < n; ++j) {
+			     for (k = 0; k < N; ++k) {
+				  R a = A[(i*fda + j) * N + k];
+				  A[(i*fda + j) * N + k] = 
+				       A[(j*fda + i) * N + k];
+				  A[(j*fda + i) * N + k] = a;
+			     }
+			}
+		   }
+	      }
+	  }
+     } else {
+	  int n2 = n / 2;
+	  rec_transpose_sq_ip_Ntuple(A, n2, fda, N);
+	  rec_transpose_sq_ip_Ntuple((A + n2*N) + n2*N*fda, n - n2, fda, N);
+	  rec_transpose_swap_Ntuple(A + n2*N, A + n2*N*fda, n2, n - n2, fda,N);
+     }
+}
+
+/*************************************************************************/
+/* In-place transposes of non-square matrices: */
+
+/* Transpose the matrix A in-place, where A is an (n*d) x (m*d) matrix
+   of N-tuples and buf contains at least n*m*d*N elements.  In
+   general, to transpose a p x q matrix, you should call this routine
+   with d = gcd(p, q), n = p/d, and m = q/d. */
+void X(transpose)(R *A, int n, int m, int d, int N, R *buf)
+{
+     A(n > 0 && m > 0 && N > 0 && d > 0);
+     if (d == 1) {
+	  rec_transpose_Ntuple(A, buf, n,m, m,n, N);
+	  memcpy(A, buf, m*n*N*sizeof(R));
+     }
+     else if (n*m == 1) {
+	  rec_transpose_sq_ip_Ntuple(A, d, d, N);
+     }
+     else {
+	  int i, num_el = n*m*d*N;
+
+	  /* treat as (d x n) x (d' x m) matrix.  (d' = d) */
+
+	  /* First, transpose d x (n x d') x m to d x (d' x n) x m,
+	     using the buf matrix.  This consists of d transposes
+	     of contiguous n x d' matrices of m-tuples. */
+	  if (n > 1) {
+	       for (i = 0; i < d; ++i) {
+		    rec_transpose_Ntuple(A + i*num_el, buf,
+					 n,d, d,n, m*N);
+		    memcpy(A + i*num_el, buf, num_el*sizeof(R));
+	       }
+	  }
+	  
+	  /* Now, transpose (d x d') x (n x m) to (d' x d) x (n x m), which
+	     is a square in-place transpose of n*m-tuples: */
+	  rec_transpose_sq_ip_Ntuple(A, d, d, n*m*N);
+
+	  /* Finally, transpose d' x ((d x n) x m) to d' x (m x (d x n)),
+	     using the buf matrix.  This consists of d' transposes
+	     of contiguous d*n x m matrices. */
+	  if (m > 1) {
+	       for (i = 0; i < d; ++i) {
+		    rec_transpose_Ntuple(A + i*num_el, buf,
+					 d*n,m, m,d*n, N);
+		    memcpy(A + i*num_el, buf, num_el*sizeof(R));
+	       }
+	  }
+     }
+}
+
+/*************************************************************************/
+/* In-place transpose routine from TOMS.  This routine is much slower
+   than the cache-oblivious algorithm above, but is has the advantage
+   of requiring less buffer space for the case of gcd(nx,ny) small. */
+
+/*
+ * TOMS Transpose.  Revised version of algorithm 380.
+ * 
+ * These routines do in-place transposes of arrays.
+ * 
+ * [ Cate, E.G. and Twigg, D.W., ACM Transactions on Mathematical Software, 
+ *   vol. 3, no. 1, 104-110 (1977) ]
+ * 
+ * C version by Steven G. Johnson. February 1997.
+ */
+
+/*
+ * "a" is a 1D array of length ny*nx*N which constains the nx x ny
+ * matrix of N-tuples to be transposed.  "a" is stored in row-major
+ * order (last index varies fastest).  move is a 1D array of length
+ * move_size used to store information to speed up the process.  The
+ * value move_size=(ny+nx)/2 is recommended.  buf should be an array
+ * of length 2*N.
+ * 
+ */
+
+void X(transpose_slow)(R *a, int nx, int ny, int N,
+		       char *move, int move_size, R *buf)
+{
+     int i, j, im, mn;
+     R *b, *c, *d;
+     int ncount;
+     int k;
+     
+     /* check arguments and initialize: */
+     A(ny > 0 && nx > 0 && N > 0 && move_size > 0);
+     
+     b = buf;
+     
+     if (ny == nx) {
+	  /*
+	   * if matrix is square, exchange elements a(i,j) and a(j,i):
+	   */
+	  for (i = 0; i < nx; ++i)
+	       for (j = i + 1; j < nx; ++j) {
+		    memcpy(b, &a[N * (i + j * nx)], N * sizeof(R));
+		    memcpy(&a[N * (i + j * nx)], &a[N * (j + i * nx)], N * sizeof(R));
+		    memcpy(&a[N * (j + i * nx)], b, N * sizeof(R));
+	       }
+	  return;
+     }
+     c = buf + N;
+     ncount = 2;		/* always at least 2 fixed points */
+     k = (mn = ny * nx) - 1;
+     
+     for (i = 0; i < move_size; ++i)
+	  move[i] = 0;
+     
+     if (ny >= 3 && nx >= 3)
+	  ncount += gcd(ny - 1, nx - 1) - 1;	/* # fixed points */
+     
+     i = 1;
+     im = ny;
+     
+     while (1) {
+	  int i1, i2, i1c, i2c;
+	  int kmi;
+	  
+	  /** Rearrange the elements of a loop
+	      and its companion loop: **/
+	  
+	  i1 = i;
+	  kmi = k - i;
+	  memcpy(b, &a[N * i1], N * sizeof(R));
+	  i1c = kmi;
+	  memcpy(c, &a[N * i1c], N * sizeof(R));
+	  
+	  while (1) {
+	       i2 = ny * i1 - k * (i1 / nx);
+	       i2c = k - i2;
+	       if (i1 < move_size)
+		    move[i1] = 1;
+	       if (i1c < move_size)
+		    move[i1c] = 1;
+	       ncount += 2;
+	       if (i2 == i)
+		    break;
+	       if (i2 == kmi) {
+		    d = b;
+		    b = c;
+		    c = d;
+		    break;
+	       }
+	       memcpy(&a[N * i1], &a[N * i2], 
+		      N * sizeof(R));
+	       memcpy(&a[N * i1c], &a[N * i2c], 
+		      N * sizeof(R));
+	       i1 = i2;
+	       i1c = i2c;
+	  }
+	  memcpy(&a[N * i1], b, N * sizeof(R));
+	  memcpy(&a[N * i1c], c, N * sizeof(R));
+	  
+	  if (ncount >= mn)
+	       break;	/* we've moved all elements */
+	  
+	  /** Search for loops to rearrange: **/
+	  
+	  while (1) {
+	       int max = k - i;
+	       ++i;
+	       A(i <= max);
+	       im += ny;
+	       if (im > k)
+		    im -= k;
+	       i2 = im;
+	       if (i == i2)
+		    continue;
+	       if (i >= move_size) {
+		    while (i2 > i && i2 < max) {
+			 i1 = i2;
+			 i2 = ny * i1 - k * (i1 / nx);
+		    }
+		    if (i2 == i)
+			 break;
+	       } else if (!move[i])
+		    break;
+	  }
+     }
+}
diff --git a/src/fftw3/kernel/trig.c b/src/fftw3/kernel/trig.c
new file mode 100644
index 0000000..0d4c3b2
--- /dev/null
+++ b/src/fftw3/kernel/trig.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: trig.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* trigonometric functions */
+#include "ifftw.h"
+#include <math.h>
+
+trigreal X(cos2pi)(int m, int n)
+{
+     return X(sincos)((trigreal)m, (trigreal)n, 0);
+}
+
+trigreal X(sin2pi)(int m, int n)
+{
+     return X(sincos)((trigreal)m, (trigreal)n, 1);
+}
+
+trigreal X(tan2pi)(int m, int n)
+{
+#if 0      /* unimplemented, unused */
+     trigreal dm = m, dn = n;
+     return TAN(by2pi(dm, dn));
+#endif
+     UNUSED(m); UNUSED(n);
+     return 0.0;
+}
diff --git a/src/fftw3/kernel/trig1.c b/src/fftw3/kernel/trig1.c
new file mode 100644
index 0000000..b86362c
--- /dev/null
+++ b/src/fftw3/kernel/trig1.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: trig1.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* trigonometric functions */
+#include "ifftw.h"
+#include <math.h>
+
+#ifdef FFTW_LDOUBLE
+#  define COS cosl
+#  define SIN sinl
+#  define TAN tanl
+#  define KTRIG(x) (x##L)
+#else
+#  define COS cos
+#  define SIN sin
+#  define TAN tan
+#  define KTRIG(x) (x)
+#endif
+
+static const trigreal K2PI =
+    KTRIG(6.2831853071795864769252867665590057683943388);
+#define by2pi(m, n) ((K2PI * (m)) / (n))
+
+/*
+ * Improve accuracy by reducing x to range [0..1/8]
+ * before multiplication by 2 * PI.
+ */
+
+trigreal X(sincos)(trigreal m, trigreal n, int sinp)
+{
+     /* waiting for C to get tail recursion... */
+     trigreal half_n = n * KTRIG(0.5);
+     trigreal quarter_n = half_n * KTRIG(0.5);
+     trigreal eighth_n = quarter_n * KTRIG(0.5);
+     trigreal sgn = KTRIG(1.0);
+
+     if (sinp) goto sin;
+ cos:
+     if (m < 0) { m = -m; /* goto cos; */ }
+     if (m > half_n) { m = n - m; goto cos; }
+     if (m > eighth_n) { m = quarter_n - m; goto sin; }
+     return sgn * COS(by2pi(m, n));
+
+ msin:
+     sgn = -sgn;
+ sin:
+     if (m < 0) { m = -m; goto msin; }
+     if (m > half_n) { m = n - m; goto msin; }
+     if (m > eighth_n) { m = quarter_n - m; goto cos; }
+     return sgn * SIN(by2pi(m, n));
+}
diff --git a/src/fftw3/kernel/twiddle.c b/src/fftw3/kernel/twiddle.c
new file mode 100644
index 0000000..785bdd7
--- /dev/null
+++ b/src/fftw3/kernel/twiddle.c
@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: twiddle.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* Twiddle manipulation */
+
+#include "ifftw.h"
+#include <math.h>
+
+/* table of known twiddle factors */
+static twid *twlist = (twid *) 0;
+
+static int equal_instr(const tw_instr *p, const tw_instr *q)
+{
+     if (p == q)
+          return 1;
+
+     for (;; ++p, ++q) {
+          if (p->op != q->op || p->v != q->v || p->i != q->i)
+               return 0;
+          if (p->op == TW_NEXT)  /* == q->op */
+               return 1;
+     }
+     A(0 /* can't happen */);
+}
+
+static int ok_twid(const twid *t, const tw_instr *q, int n, int r, int m)
+{
+     return (n == t->n && r == t->r && m <= t->m && equal_instr(t->instr, q));
+}
+
+static twid *lookup(const tw_instr *q, int n, int r, int m)
+{
+     twid *p;
+
+     for (p = twlist; p && !ok_twid(p, q, n, r, m); p = p->cdr)
+          ;
+     return p;
+}
+
+static int twlen0(int r, const tw_instr **pp)
+{
+     int ntwiddle = 0;
+     const tw_instr *p = *pp;
+
+     /* compute length of bytecode program */
+     A(r > 0);
+     for ( ; p->op != TW_NEXT; ++p) {
+	  switch (p->op) {
+	      case TW_FULL:
+		   ntwiddle += (r - 1) * 2;
+		   break;
+	      case TW_GENERIC:
+		   ntwiddle += r * 2;
+		   break;
+	      default:
+		   ++ntwiddle;
+	  }
+     }
+
+     *pp = p;
+     return ntwiddle;
+}
+
+int X(twiddle_length)(int r, const tw_instr *p)
+{
+     return twlen0(r, &p);
+}
+
+static R *compute(const tw_instr *instr, int n, int r, int m)
+{
+     int ntwiddle, j;
+     R *W, *W0;
+     const tw_instr *p;
+
+     static trigreal (*const f[])(int, int) = { 
+	  X(cos2pi), X(sin2pi), X(tan2pi) 
+     };
+
+     p = instr;
+     ntwiddle = twlen0(r, &p);
+
+     W0 = W = (R *)MALLOC(ntwiddle * (m / p->v) * sizeof(R), TWIDDLES);
+
+     for (j = 0; j < m; j += p->v) {
+          for (p = instr; p->op != TW_NEXT; ++p) {
+	       switch (p->op) {
+		   case TW_FULL:
+		   {
+			int i;
+			A((int)p->i == r); /* consistency check */
+			for (i = 1; i < r; ++i) {
+			     *W++ = f[TW_COS]((j + p->v) * i, n);
+			     *W++ = f[TW_SIN]((j + p->v) * i, n);
+			}
+			break;
+		   }
+
+		   case TW_GENERIC:
+		   {
+			int i;
+			A(p->v == 0); /* unused */
+			A(p->i == 0); /* unused */
+			for (i = 0; i < r; ++i) {
+			     int k = j * r + i;
+			     *W++ = f[TW_COS](k, n);
+			     *W++ = FFT_SIGN * f[TW_SIN](k, n);
+			}
+			break;
+		   }
+		   
+		   default:
+			*W++ = f[p->op](((signed int)(j + p->v)) * p->i, n);
+			break;
+	       }
+	  }
+          A(m % p->v == 0);
+     }
+
+     return W0;
+}
+
+void X(mktwiddle)(twid **pp, const tw_instr *instr, int n, int r, int m)
+{
+     twid *p;
+
+     if (*pp) return;  /* already created */
+
+     if ((p = lookup(instr, n, r, m))) {
+          ++p->refcnt;
+	  goto done;
+     }
+
+     p = (twid *) MALLOC(sizeof(twid), TWIDDLES);
+     p->n = n;
+     p->r = r;
+     p->m = m;
+     p->instr = instr;
+     p->refcnt = 1;
+     p->W = compute(instr, n, r, m);
+
+     /* cons! onto twlist */
+     p->cdr = twlist;
+     twlist = p;
+
+ done:
+     *pp = p;
+     return;
+}
+
+void X(twiddle_destroy)(twid **pp)
+{
+     twid *p = *pp;
+     if (p) {
+          twid **q;
+          if ((--p->refcnt) == 0) {
+               /* remove p from twiddle list */
+               for (q = &twlist; *q; q = &((*q)->cdr)) {
+                    if (*q == p) {
+                         *q = p->cdr;
+                         X(ifree)(p->W);
+                         X(ifree)(p);
+			 goto done;
+                    }
+               }
+               A(0 /* can't happen */ );
+          }
+     }
+ done:
+     *pp = 0; /* destroy pointer */
+     return;
+}
+
+
+void X(twiddle_awake)(int flg, twid **pp, 
+		      const tw_instr *instr, int n, int r, int m)
+{
+     if (flg) 
+	  X(mktwiddle)(pp, instr, n, r, m);
+     else 
+	  X(twiddle_destroy)(pp);
+}
diff --git a/src/fftw3/rdft/buffered2.c b/src/fftw3/rdft/buffered2.c
new file mode 100644
index 0000000..54fc7bc
--- /dev/null
+++ b/src/fftw3/rdft/buffered2.c
@@ -0,0 +1,418 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: buffered2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+typedef struct {
+     int nbuf;
+     int maxbufsz;
+     int skew_alignment;
+     int skew;
+     const char *nam;
+} bufadt;
+
+typedef struct {
+     solver super;
+     const bufadt *adt;
+} S;
+
+typedef struct {
+     plan_rdft2 super;
+
+     plan *cld, *cldrest;
+     int n, vl, nbuf, bufdist;
+     int os, ivs, ovs;
+
+     const S *slv;
+} P;
+
+/***************************************************************************/
+
+/* FIXME: have alternate copy functions that push a vector loop inside
+   the n loops? */
+
+/* copy halfcomplex array r (contiguous) to complex (strided) array rio/iio. */
+static void hc2c(int n, R *r, R *rio, R *iio, int os)
+{
+     int n2 = (n + 1) / 2;
+     int i;
+
+     rio[0] = r[0];
+     iio[0] = 0;
+     for (i = 1; i < ((n2 - 1) & 3) + 1; ++i) {
+	  rio[i * os] = r[i];
+	  iio[i * os] = r[n - i];
+     }
+     for (; i < n2; i += 4) {
+	  R r0, r1, r2, r3;
+	  R i0, i1, i2, i3;
+	  r0 = r[i];
+	  r1 = r[i + 1];
+	  r2 = r[i + 2];
+	  r3 = r[i + 3];
+	  i3 = r[n - (i + 3)];
+	  i2 = r[n - (i + 2)];
+	  i1 = r[n - (i + 1)];
+	  i0 = r[n - i];
+	  rio[i * os] = r0;
+	  iio[i * os] = i0;
+	  rio[(i + 1) * os] = r1;
+	  iio[(i + 1) * os] = i1;
+	  rio[(i + 2) * os] = r2;
+	  iio[(i + 2) * os] = i2;
+	  rio[(i + 3) * os] = r3;
+	  iio[(i + 3) * os] = i3;
+     }
+     if ((n & 1) == 0) {	/* store the Nyquist frequency */
+	  rio[n2 * os] = r[n2];
+	  iio[n2 * os] = 0.0;
+     }
+}
+
+/* reverse of hc2c */
+static void c2hc(int n, R *rio, R *iio, int is, R *r)
+{
+     int n2 = (n + 1) / 2;
+     int i;
+
+     r[0] = rio[0];
+     for (i = 1; i < ((n2 - 1) & 3) + 1; ++i) {
+	  r[i] = rio[i * is];
+	  r[n - i] = iio[i * is];
+     }
+     for (; i < n2; i += 4) {
+	  R r0, r1, r2, r3;
+	  R i0, i1, i2, i3;
+	  r0 = rio[i * is];
+	  i0 = iio[i * is];
+	  r1 = rio[(i + 1) * is];
+	  i1 = iio[(i + 1) * is];
+	  r2 = rio[(i + 2) * is];
+	  i2 = iio[(i + 2) * is];
+	  r3 = rio[(i + 3) * is];
+	  i3 = iio[(i + 3) * is];
+	  r[i] = r0;
+	  r[i + 1] = r1;
+	  r[i + 2] = r2;
+	  r[i + 3] = r3;
+	  r[n - (i + 3)] = i3;
+	  r[n - (i + 2)] = i2;
+	  r[n - (i + 1)] = i1;
+	  r[n - i] = i0;
+     }
+     if ((n & 1) == 0)		/* store the Nyquist frequency */
+	  r[n2] = rio[n2 * is];
+}
+
+/***************************************************************************/
+
+static void apply_r2hc(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+     plan_rdft *cld = (plan_rdft *) ego->cld;
+     int i, j, vl = ego->vl, nbuf = ego->nbuf, bufdist = ego->bufdist;
+     int n = ego->n;
+     int ivs = ego->ivs, ovs = ego->ovs, os = ego->os;
+     R *bufs;
+
+     bufs = (R *)MALLOC(sizeof(R) * nbuf * bufdist, BUFFERS);
+
+     for (i = nbuf; i <= vl; i += nbuf) {
+          /* transform to bufs: */
+          cld->apply((plan *) cld, r, bufs);
+	  r += ivs;
+
+          /* copy back */
+	  for (j = 0; j < nbuf; ++j, rio += ovs, iio += ovs)
+	       hc2c(n, bufs + j*bufdist, rio, iio, os);
+     }
+
+     /* Do the remaining transforms, if any: */
+     {
+	  plan_rdft *cldrest = (plan_rdft *) ego->cldrest;
+	  R *b = bufs;
+	  cldrest->apply((plan *) cldrest, r, bufs);
+	  for (i -= nbuf; i < vl; ++i, rio += ovs, iio += ovs, b += bufdist)
+	       hc2c(n, b, rio, iio, os);
+     }
+
+     X(ifree)(bufs);
+}
+
+static void apply_hc2r(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+     plan_rdft *cld = (plan_rdft *) ego->cld;
+     int i, j, vl = ego->vl, nbuf = ego->nbuf, bufdist = ego->bufdist;
+     int n = ego->n;
+     int ivs = ego->ivs, ovs = ego->ovs, is = ego->os;
+     R *bufs;
+
+     bufs = (R *)MALLOC(sizeof(R) * nbuf * bufdist, BUFFERS);
+
+     for (i = nbuf; i <= vl; i += nbuf) {
+          /* copy to bufs */
+	  for (j = 0; j < nbuf; ++j, rio += ivs, iio += ivs)
+	       c2hc(n, rio, iio, is, bufs + j*bufdist);
+
+          /* transform back: */
+          cld->apply((plan *) cld, bufs, r);
+	  r += ovs;
+     }
+
+     /* Do the remaining transforms, if any: */
+     {
+	  plan_rdft *cldrest;
+	  R *b = bufs;
+	  for (i -= nbuf; i < vl; ++i, rio += ivs, iio += ivs, b += bufdist)
+	       c2hc(n, rio, iio, is, b);
+	  cldrest = (plan_rdft *) ego->cldrest;
+	  cldrest->apply((plan *) cldrest, bufs, r);
+     }
+
+     X(ifree)(bufs);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+
+     AWAKE(ego->cld, flg);
+     AWAKE(ego->cldrest, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cldrest);
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(%s-%s-%d%v/%d-%d%(%p%)%(%p%))",
+              ego->slv->adt->nam,
+	      ego->super.apply == apply_r2hc ? "r2hc" : "hc2r",
+              ego->n, ego->nbuf,
+              ego->vl, ego->bufdist % ego->n,
+              ego->cld, ego->cldrest);
+}
+
+static int min_nbuf(const problem_rdft2 *p, int n, int vl)
+{
+     int is, os, ivs, ovs;
+
+     if (p->r != p->rio && p->r != p->iio)
+	  return 1;
+     if (X(rdft2_inplace_strides(p, RNK_MINFTY)))
+	  return 1;
+     A(p->vecsz->rnk == 1); /*  rank 0 and MINFTY are inplace */
+
+     X(rdft2_strides)(p->kind, p->sz->dims, &is, &os);
+     X(rdft2_strides)(p->kind, p->vecsz->dims, &ivs, &ovs);
+     
+     /* handle one potentially common case: "contiguous" real and
+	complex arrays, which overlap because of the differing sizes. */
+     if (n * X(iabs)(is) <= X(iabs)(ivs)
+	 && (n/2 + 1) * X(iabs)(os) <= X(iabs)(ovs)
+	 && X(iabs)((int) (p->rio - p->iio)) <= X(iabs)(os)
+	 && ivs > 0 && ovs > 0) {
+	  int vsmin = X(imin)(ivs, ovs);
+	  int vsmax = X(imax)(ivs, ovs);
+	  return(((vsmax - vsmin) * vl + vsmin - 1) / vsmin);
+     }
+
+     return vl; /* punt: just buffer the whole vector */
+}
+
+static int compute_nbuf(int n, int vl, const S *ego)
+{
+     return X(compute_nbuf)(n, vl, ego->adt->nbuf, ego->adt->maxbufsz);
+}
+
+static int toobig(int n, const S *ego)
+{
+     return (n > ego->adt->maxbufsz);
+}
+
+static int applicable0(const problem *p_, const S *ego, const planner *plnr)
+{
+     UNUSED(ego);
+     if (RDFT2P(p_)) {
+          const problem_rdft2 *p = (const problem_rdft2 *) p_;
+	  return(p->vecsz->rnk <= 1 && p->sz->rnk == 1
+		 && !(toobig(p->sz->dims[0].n, ego) && CONSERVE_MEMORYP(plnr)));
+     }
+     return 0;
+}
+
+static int applicable(const problem *p_, const S *ego, const planner *plnr)
+{
+     const problem_rdft2 *p;
+
+     if (NO_BUFFERINGP(plnr)) return 0;
+     if (!applicable0(p_, ego, plnr)) return 0;
+
+     p = (const problem_rdft2 *) p_;
+     if (NO_UGLYP(plnr)) {
+	  if (p->r != p->rio && p->r != p->iio) return 0;
+	  if (toobig(p->sz->dims[0].n, ego)) return 0;
+     }
+     return 1;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     const bufadt *adt = ego->adt;
+     P *pln;
+     plan *cld = (plan *) 0;
+     plan *cldrest = (plan *) 0;
+     problem *cldp = 0;
+     const problem_rdft2 *p = (const problem_rdft2 *) p_;
+     R *bufs = (R *) 0;
+     int nbuf = 0, bufdist, n, vl;
+     int ivs, ovs;
+
+     static const plan_adt padt = {
+	  X(rdft2_solve), awake, print, destroy
+     };
+
+
+     if (!applicable(p_, ego, plnr))
+          goto nada;
+
+     n = p->sz->dims[0].n;
+     X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs);
+
+     nbuf = X(imax)(compute_nbuf(n, vl, ego), min_nbuf(p, n, vl));
+     A(nbuf > 0);
+
+     /*
+      * Determine BUFDIST, the offset between successive array bufs.
+      * bufdist = n + skew, where skew is chosen such that bufdist %
+      * skew_alignment = skew.
+      */
+     if (vl == 1) {
+          bufdist = n;
+     } else {
+          bufdist =
+               n + ((adt->skew_alignment + adt->skew - n % adt->skew_alignment)
+                    % adt->skew_alignment);
+          A(p->vecsz->rnk == 1);
+     }
+
+     /* initial allocation for the purpose of planning */
+     bufs = (R *) MALLOC(sizeof(R) * nbuf * bufdist, BUFFERS);
+
+     if (p->kind == R2HC)
+	  cldp =
+	       X(mkproblem_rdft_d)(
+		    X(mktensor_1d)(n, p->sz->dims[0].is, 1),
+		    X(mktensor_1d)(nbuf, ivs, bufdist),
+		    TAINT(p->r, ivs * nbuf), bufs, &p->kind);
+     else {
+	  A(p->kind == HC2R);
+	  plnr->problem_flags |= DESTROY_INPUT; /* always ok to destroy buf */
+	  cldp =
+	       X(mkproblem_rdft_d)(
+		    X(mktensor_1d)(n, 1, p->sz->dims[0].os),
+		    X(mktensor_1d)(nbuf, bufdist, ovs),
+		    bufs, TAINT(p->r, ovs * nbuf), &p->kind);
+     }
+     if (!(cld = X(mkplan_d)(plnr, cldp))) goto nada;
+
+     /* plan the leftover transforms (cldrest): */
+     if (p->kind == R2HC)
+	  cldp =
+	       X(mkproblem_rdft_d)(
+		    X(mktensor_1d)(n, p->sz->dims[0].is, 1),
+		    X(mktensor_1d)(vl % nbuf, ivs, bufdist),
+		    p->r + ivs * (nbuf * (vl / nbuf)), bufs, &p->kind);
+     else /* HC2R */
+	  cldp =
+	       X(mkproblem_rdft_d)(
+		    X(mktensor_1d)(n, 1, p->sz->dims[0].os),
+		    X(mktensor_1d)(vl % nbuf, bufdist, ovs),
+			 bufs, p->r + ovs * (nbuf * (vl / nbuf)), &p->kind);
+     if (!(cldrest = X(mkplan_d)(plnr, cldp))) goto nada;
+
+     /* deallocate buffers, let apply() allocate them for real */
+     X(ifree)(bufs);
+     bufs = 0;
+
+     pln = MKPLAN_RDFT2(P, &padt, p->kind == R2HC ? apply_r2hc : apply_hc2r);
+     pln->cld = cld;
+     pln->cldrest = cldrest;
+     pln->slv = ego;
+     pln->n = n;
+     pln->vl = vl;
+     if (p->kind == R2HC) {
+	  pln->ivs = ivs * nbuf;
+	  pln->ovs = ovs;
+	  pln->os = p->sz->dims[0].os; /* stride of rio/iio  */
+     }
+     else { /* HC2R */
+	  pln->ivs = ivs;
+	  pln->ovs = ovs * nbuf;
+	  pln->os = p->sz->dims[0].is; /* stride of rio/iio  */
+     }
+
+
+     pln->nbuf = nbuf;
+     pln->bufdist = bufdist;
+
+     X(ops_madd)(vl / nbuf, &cld->ops, &cldrest->ops,
+		 &pln->super.super.ops);
+     pln->super.super.ops.other += (p->kind == R2HC ? (n + 2) : n) * vl;
+
+     return &(pln->super.super);
+
+ nada:
+     X(ifree0)(bufs);
+     X(plan_destroy_internal)(cldrest);
+     X(plan_destroy_internal)(cld);
+     return (plan *) 0;
+}
+
+static solver *mksolver(const bufadt *adt)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->adt = adt;
+     return &(slv->super);
+}
+
+void X(rdft2_buffered_register)(planner *p)
+{
+     /* FIXME: what are good defaults? */
+     static const bufadt adt = {
+	  /* nbuf */           8,
+	  /* maxbufsz */       (65536 / sizeof(R)),
+	  /* skew_alignment */ 8,
+	  /* skew */           5,
+	  /* nam */            "rdft2-buffered"
+     };
+
+     REGISTER_SOLVER(p, mksolver(&adt));
+}
diff --git a/src/fftw3/rdft/codelet-rdft.h b/src/fftw3/rdft/codelet-rdft.h
new file mode 100644
index 0000000..27d76a8
--- /dev/null
+++ b/src/fftw3/rdft/codelet-rdft.h
@@ -0,0 +1,192 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/*
+ * This header file must include every file or define every
+ * type or macro which is required to compile a codelet.
+ */
+
+#ifndef __RDFT_CODELET_H__
+#define __RDFT_CODELET_H__
+
+#include "ifftw.h"
+
+/**************************************************************
+ * types of codelets
+ **************************************************************/
+
+/* FOOab, with a,b in {0,1}, denotes the FOO transform
+   where a/b say whether the input/output are shifted by
+   half a sample/slot. */
+
+typedef enum {
+     R2HC00, R2HC01, R2HC10, R2HC11,
+     HC2R00, HC2R01, HC2R10, HC2R11,
+     DHT, 
+     REDFT00, REDFT01, REDFT10, REDFT11, /* real-even == DCT's */
+     RODFT00, RODFT01, RODFT10, RODFT11  /*  real-odd == DST's */
+} rdft_kind;
+
+/* standard R2HC/HC2R transforms are unshifted */
+#define R2HC R2HC00
+#define HC2R HC2R00
+
+#define R2HCII R2HC01
+#define HC2RIII HC2R10
+
+/* (k) >= R2HC00 produces a warning under gcc because checking x >= 0
+   is superfluous for unsigned values...but it is needed because other
+   compilers (e.g. icc) may define the enum to be a signed int...grrr. */
+#define R2HC_KINDP(k) ((k) >= R2HC00 && (k) <= R2HC11) /* uses kr2hc_genus */
+#define HC2R_KINDP(k) ((k) >= HC2R00 && (k) <= HC2R11) /* uses khc2r_genus */
+
+#define R2R_KINDP(k) ((k) >= DHT) /* uses kr2r_genus */
+
+#define REDFT_KINDP(k) ((k) >= REDFT00 && (k) <= REDFT11)
+#define RODFT_KINDP(k) ((k) >= RODFT00 && (k) <= RODFT11)
+#define REODFT_KINDP(k) ((k) >= REDFT00 && (k) <= RODFT11)
+
+/* real-input DFT codelets */
+typedef struct kr2hc_desc_s kr2hc_desc;
+
+typedef struct {
+     int (*okp)(
+	  const kr2hc_desc *desc,
+	  const R *I, const R *ro, const R *io,
+	  int is, int ios, int ros, int vl, int ivs, int ovs);
+     rdft_kind kind;
+     int vl;
+} kr2hc_genus;
+
+struct kr2hc_desc_s {
+     int sz;    /* size of transform computed */
+     const char *nam;
+     opcnt ops;
+     const kr2hc_genus *genus;
+     int is;
+     int ros, ios;
+     int ivs;
+     int ovs;
+};
+
+typedef void (*kr2hc) (const R *I, R *ro, R *io, stride is,
+		       stride ros, stride ios, int vl, int ivs, int ovs);
+void X(kr2hc_register)(planner *p, kr2hc codelet, const kr2hc_desc *desc);
+
+/* real-input DFT codelets, type II (middle case of hc2hc DIT) */
+typedef kr2hc_desc kr2hcII_desc;
+typedef kr2hc_genus kr2hcII_genus;
+typedef kr2hc kr2hcII;
+#define kr2hcII_register kr2hc_register
+
+/* half-complex to half-complex DIT/DIF codelets: */
+typedef struct hc2hc_desc_s hc2hc_desc;
+
+typedef struct {
+     int (*okp)(
+	  const struct hc2hc_desc_s *desc,
+	  const R *rio, const R *iio, int ios, int vs, int m, int dist);
+     rdft_kind kind;
+     int vl;
+} hc2hc_genus;
+
+struct hc2hc_desc_s {
+     int radix;
+     const char *nam;
+     const tw_instr *tw;
+     opcnt ops;
+     const hc2hc_genus *genus;
+     int s1;
+     int s2;
+     int dist;
+};
+
+typedef const R *(*khc2hc) (R *rioarray, R *iioarray, const R *W,
+                              stride ios, int m, int dist);
+void X(khc2hc_dit_register)(planner *p, khc2hc codelet, const hc2hc_desc *desc);
+
+extern const solvtab X(solvtab_rdft_r2hc);
+
+/* real-output DFT codelets */
+typedef struct khc2r_desc_s khc2r_desc;
+
+typedef struct {
+     int (*okp)(
+	  const khc2r_desc *desc,
+	  const R *ri, const R *ii, const R *O,
+	  int ris, int iis, int os, int vl, int ivs, int ovs);
+     rdft_kind kind;
+     int vl;
+} khc2r_genus;
+
+struct khc2r_desc_s {
+     int sz;    /* size of transform computed */
+     const char *nam;
+     opcnt ops;
+     const khc2r_genus *genus;
+     int ris, iis;
+     int os;
+     int ivs;
+     int ovs;
+};
+
+typedef void (*khc2r) (const R *ri, const R *ii, R *O, stride ris,
+		       stride iis, stride os, int vl, int ivs, int ovs);
+void X(khc2r_register)(planner *p, khc2r codelet, const khc2r_desc *desc);
+
+/* real-output DFT codelets, type III (middle case of hc2hc DIF) */
+typedef khc2r_desc khc2rIII_desc;
+typedef khc2r_genus khc2rIII_genus;
+typedef khc2r khc2rIII;
+#define khc2rIII_register khc2r_register
+
+void X(khc2hc_dif_register)(planner *p, khc2hc codelet, const hc2hc_desc *desc);
+
+extern const solvtab X(solvtab_rdft_hc2r);
+
+/* real-input & output DFT-like codelets (DHT, etc.) */
+typedef struct kr2r_desc_s kr2r_desc;
+
+typedef struct {
+     int (*okp)(
+	  const kr2r_desc *desc,
+	  const R *I, const R *O,
+	  int is, int os, int vl, int ivs, int ovs);
+     int vl;
+} kr2r_genus;
+
+struct kr2r_desc_s {
+     int sz;    /* size of transform computed */
+     const char *nam;
+     opcnt ops;
+     const kr2r_genus *genus;
+     rdft_kind kind;
+     int is, os;
+     int ivs;
+     int ovs;
+};
+
+typedef void (*kr2r) (const R *I, R *O, stride is, stride os,
+		      int vl, int ivs, int ovs);
+void X(kr2r_register)(planner *p, kr2r codelet, const kr2r_desc *desc);
+
+extern const solvtab X(solvtab_rdft_r2r);
+
+#endif				/* __RDFT_CODELET_H__ */
diff --git a/src/fftw3/rdft/codelets/hb.h b/src/fftw3/rdft/codelets/hb.h
new file mode 100644
index 0000000..bb8f528
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hb.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+
+#define GENUS X(rdft_hb_genus)
+extern const hc2hc_genus GENUS;
diff --git a/src/fftw3/rdft/codelets/hc2r.c b/src/fftw3/rdft/codelets/hc2r.c
new file mode 100644
index 0000000..7daf21c
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "codelet-rdft.h"
+#include "hc2r.h"
+
+static int okp(const khc2r_desc *d,
+	       const R *ri, const R *ii,
+	       const R *O,
+	       int ris, int iis, int os, int vl, int ivs, int ovs)
+{
+     UNUSED(ri); UNUSED(ii); UNUSED(O); UNUSED(vl);
+     return (1
+	     && (!d->ris || (d->ris == ris))
+	     && (!d->iis || (d->ris == iis))
+	     && (!d->os || (d->os == os))
+	     && (!d->ivs || (d->ivs == ivs))
+	     && (!d->ovs || (d->ovs == ovs))
+	  );
+}
+
+const khc2r_genus GENUS = { okp, HC2R, 1 };
+
+#undef GENUS
+#include "hc2rIII.h"
+
+const khc2rIII_genus GENUS = { okp, HC2RIII, 1 };
diff --git a/src/fftw3/rdft/codelets/hc2r.h b/src/fftw3/rdft/codelets/hc2r.h
new file mode 100644
index 0000000..3dd17be
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+
+#define GENUS X(rdft_hc2r_genus)
+extern const khc2r_genus GENUS;
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_10.c b/src/fftw3/rdft/codelets/hc2r/hb_10.c
new file mode 100644
index 0000000..3d96502
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_10.c
@@ -0,0 +1,265 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:33 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 10 -dif -name hb_10 -include hb.h */
+
+/*
+ * This function contains 102 FP additions, 60 FP multiplications,
+ * (or, 72 additions, 30 multiplications, 30 fused multiply/add),
+ * 39 stack variables, and 40 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_10.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_10.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_10.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_10(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 18) {
+	  E T3, Tk, Tw, T1w, TO, TP, T1E, T1D, Tr, TX, Ti, T1l, TZ, T10, T1s;
+	  E T1p, T1z, T1B, TL, TS;
+	  {
+	       E T1, T2, Tu, Tv;
+	       T1 = rio[0];
+	       T2 = iio[-WS(ios, 5)];
+	       T3 = T1 + T2;
+	       Tk = T1 - T2;
+	       Tu = iio[0];
+	       Tv = rio[WS(ios, 5)];
+	       Tw = Tu + Tv;
+	       T1w = Tu - Tv;
+	  }
+	  {
+	       E T6, Tl, Tg, Tp, T9, Tm, Td, To;
+	       {
+		    E T4, T5, Te, Tf;
+		    T4 = rio[WS(ios, 2)];
+		    T5 = iio[-WS(ios, 7)];
+		    T6 = T4 + T5;
+		    Tl = T4 - T5;
+		    Te = iio[-WS(ios, 6)];
+		    Tf = rio[WS(ios, 1)];
+		    Tg = Te + Tf;
+		    Tp = Te - Tf;
+	       }
+	       {
+		    E T7, T8, Tb, Tc;
+		    T7 = iio[-WS(ios, 8)];
+		    T8 = rio[WS(ios, 3)];
+		    T9 = T7 + T8;
+		    Tm = T7 - T8;
+		    Tb = rio[WS(ios, 4)];
+		    Tc = iio[-WS(ios, 9)];
+		    Td = Tb + Tc;
+		    To = Tb - Tc;
+	       }
+	       TO = Tl - Tm;
+	       TP = To - Tp;
+	       T1E = Td - Tg;
+	       T1D = T6 - T9;
+	       {
+		    E Tn, Tq, Ta, Th;
+		    Tn = Tl + Tm;
+		    Tq = To + Tp;
+		    Tr = Tn + Tq;
+		    TX = KP559016994 * (Tn - Tq);
+		    Ta = T6 + T9;
+		    Th = Td + Tg;
+		    Ti = Ta + Th;
+		    T1l = KP559016994 * (Ta - Th);
+	       }
+	  }
+	  {
+	       E Tz, T1n, TJ, T1r, TC, T1o, TG, T1q;
+	       {
+		    E Tx, Ty, TH, TI;
+		    Tx = iio[-WS(ios, 2)];
+		    Ty = rio[WS(ios, 7)];
+		    Tz = Tx + Ty;
+		    T1n = Tx - Ty;
+		    TH = rio[WS(ios, 6)];
+		    TI = iio[-WS(ios, 1)];
+		    TJ = TH + TI;
+		    T1r = TI - TH;
+	       }
+	       {
+		    E TA, TB, TE, TF;
+		    TA = rio[WS(ios, 8)];
+		    TB = iio[-WS(ios, 3)];
+		    TC = TA + TB;
+		    T1o = TB - TA;
+		    TE = iio[-WS(ios, 4)];
+		    TF = rio[WS(ios, 9)];
+		    TG = TE + TF;
+		    T1q = TE - TF;
+	       }
+	       TZ = Tz + TC;
+	       T10 = TG + TJ;
+	       T1s = T1q - T1r;
+	       T1p = T1n - T1o;
+	       {
+		    E T1x, T1y, TD, TK;
+		    T1x = T1n + T1o;
+		    T1y = T1q + T1r;
+		    T1z = T1x + T1y;
+		    T1B = KP559016994 * (T1x - T1y);
+		    TD = Tz - TC;
+		    TK = TG - TJ;
+		    TL = TD + TK;
+		    TS = KP559016994 * (TD - TK);
+	       }
+	  }
+	  rio[0] = T3 + Ti;
+	  iio[-WS(ios, 9)] = T1w + T1z;
+	  {
+	       E Ts, TM, Tj, Tt;
+	       Ts = Tk + Tr;
+	       TM = Tw + TL;
+	       Tj = W[8];
+	       Tt = W[9];
+	       rio[WS(ios, 5)] = FNMS(Tt, TM, Tj * Ts);
+	       iio[-WS(ios, 4)] = FMA(Tt, Ts, Tj * TM);
+	  }
+	  {
+	       E T1t, T1F, T1Q, T1N, T1C, T1R, T1m, T1M, T1A, T1k;
+	       T1t = FNMS(KP951056516, T1s, KP587785252 * T1p);
+	       T1F = FNMS(KP951056516, T1E, KP587785252 * T1D);
+	       T1Q = FMA(KP951056516, T1D, KP587785252 * T1E);
+	       T1N = FMA(KP951056516, T1p, KP587785252 * T1s);
+	       T1A = FNMS(KP250000000, T1z, T1w);
+	       T1C = T1A - T1B;
+	       T1R = T1B + T1A;
+	       T1k = FNMS(KP250000000, Ti, T3);
+	       T1m = T1k - T1l;
+	       T1M = T1l + T1k;
+	       {
+		    E T1u, T1G, T1j, T1v;
+		    T1u = T1m + T1t;
+		    T1G = T1C - T1F;
+		    T1j = W[14];
+		    T1v = W[15];
+		    rio[WS(ios, 8)] = FNMS(T1v, T1G, T1j * T1u);
+		    iio[-WS(ios, 1)] = FMA(T1v, T1u, T1j * T1G);
+	       }
+	       {
+		    E T1U, T1W, T1T, T1V;
+		    T1U = T1M + T1N;
+		    T1W = T1R - T1Q;
+		    T1T = W[6];
+		    T1V = W[7];
+		    rio[WS(ios, 4)] = FNMS(T1V, T1W, T1T * T1U);
+		    iio[-WS(ios, 5)] = FMA(T1V, T1U, T1T * T1W);
+	       }
+	       {
+		    E T1I, T1K, T1H, T1J;
+		    T1I = T1m - T1t;
+		    T1K = T1F + T1C;
+		    T1H = W[2];
+		    T1J = W[3];
+		    rio[WS(ios, 2)] = FNMS(T1J, T1K, T1H * T1I);
+		    iio[-WS(ios, 7)] = FMA(T1J, T1I, T1H * T1K);
+	       }
+	       {
+		    E T1O, T1S, T1L, T1P;
+		    T1O = T1M - T1N;
+		    T1S = T1Q + T1R;
+		    T1L = W[10];
+		    T1P = W[11];
+		    rio[WS(ios, 6)] = FNMS(T1P, T1S, T1L * T1O);
+		    iio[-WS(ios, 3)] = FMA(T1P, T1O, T1L * T1S);
+	       }
+	  }
+	  {
+	       E TQ, T11, T1c, T19, TY, T18, TT, T1d, TW, TR;
+	       TQ = FNMS(KP951056516, TP, KP587785252 * TO);
+	       T11 = FNMS(KP951056516, T10, KP587785252 * TZ);
+	       T1c = FMA(KP951056516, TO, KP587785252 * TP);
+	       T19 = FMA(KP951056516, TZ, KP587785252 * T10);
+	       TW = FNMS(KP250000000, Tr, Tk);
+	       TY = TW - TX;
+	       T18 = TX + TW;
+	       TR = FNMS(KP250000000, TL, Tw);
+	       TT = TR - TS;
+	       T1d = TS + TR;
+	       {
+		    E TU, T12, TN, TV;
+		    TU = TQ + TT;
+		    T12 = TY - T11;
+		    TN = W[12];
+		    TV = W[13];
+		    iio[-WS(ios, 2)] = FMA(TN, TU, TV * T12);
+		    rio[WS(ios, 7)] = FNMS(TV, TU, TN * T12);
+	       }
+	       {
+		    E T1g, T1i, T1f, T1h;
+		    T1g = T1d - T1c;
+		    T1i = T18 + T19;
+		    T1f = W[16];
+		    T1h = W[17];
+		    iio[0] = FMA(T1f, T1g, T1h * T1i);
+		    rio[WS(ios, 9)] = FNMS(T1h, T1g, T1f * T1i);
+	       }
+	       {
+		    E T14, T16, T13, T15;
+		    T14 = TY + T11;
+		    T16 = TT - TQ;
+		    T13 = W[4];
+		    T15 = W[5];
+		    rio[WS(ios, 3)] = FNMS(T15, T16, T13 * T14);
+		    iio[-WS(ios, 6)] = FMA(T13, T16, T15 * T14);
+	       }
+	       {
+		    E T1a, T1e, T17, T1b;
+		    T1a = T18 - T19;
+		    T1e = T1c + T1d;
+		    T17 = W[0];
+		    T1b = W[1];
+		    rio[WS(ios, 1)] = FNMS(T1b, T1e, T17 * T1a);
+		    iio[-WS(ios, 8)] = FMA(T17, T1e, T1b * T1a);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 10},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 10, "hb_10", twinstr, {72, 30, 30, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_10) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_10, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_12.c b/src/fftw3/rdft/codelets/hc2r/hb_12.c
new file mode 100644
index 0000000..55e7db0
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_12.c
@@ -0,0 +1,299 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:36 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 12 -dif -name hb_12 -include hb.h */
+
+/*
+ * This function contains 118 FP additions, 60 FP multiplications,
+ * (or, 88 additions, 30 multiplications, 30 fused multiply/add),
+ * 39 stack variables, and 48 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_12.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_12.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_12.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_12(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 22) {
+	  E T5, Tt, T12, T1M, T1i, T1U, Tl, TM, T1c, T1Y, T1s, T1Q, Ta, Ty, T15;
+	  E T1N, T1l, T1V, Tg, TH, T19, T1X, T1p, T1P;
+	  {
+	       E T1, Tp, T4, T1g, Ts, T11, T10, T1h;
+	       T1 = rio[0];
+	       Tp = iio[0];
+	       {
+		    E T2, T3, Tq, Tr;
+		    T2 = rio[WS(ios, 4)];
+		    T3 = iio[-WS(ios, 8)];
+		    T4 = T2 + T3;
+		    T1g = KP866025403 * (T2 - T3);
+		    Tq = rio[WS(ios, 8)];
+		    Tr = iio[-WS(ios, 4)];
+		    Ts = Tq - Tr;
+		    T11 = KP866025403 * (Tq + Tr);
+	       }
+	       T5 = T1 + T4;
+	       Tt = Tp - Ts;
+	       T10 = FNMS(KP500000000, T4, T1);
+	       T12 = T10 - T11;
+	       T1M = T10 + T11;
+	       T1h = FMA(KP500000000, Ts, Tp);
+	       T1i = T1g + T1h;
+	       T1U = T1h - T1g;
+	  }
+	  {
+	       E Th, TL, Tk, T1a, TK, T1r, T1b, T1q;
+	       Th = iio[-WS(ios, 9)];
+	       TL = rio[WS(ios, 9)];
+	       {
+		    E Ti, Tj, TI, TJ;
+		    Ti = rio[WS(ios, 1)];
+		    Tj = rio[WS(ios, 5)];
+		    Tk = Ti + Tj;
+		    T1a = KP866025403 * (Ti - Tj);
+		    TI = iio[-WS(ios, 5)];
+		    TJ = iio[-WS(ios, 1)];
+		    TK = TI + TJ;
+		    T1r = KP866025403 * (TI - TJ);
+	       }
+	       Tl = Th + Tk;
+	       TM = TK - TL;
+	       T1b = FMA(KP500000000, TK, TL);
+	       T1c = T1a - T1b;
+	       T1Y = T1a + T1b;
+	       T1q = FNMS(KP500000000, Tk, Th);
+	       T1s = T1q + T1r;
+	       T1Q = T1q - T1r;
+	  }
+	  {
+	       E T6, Tx, T9, T1j, Tw, T14, T13, T1k;
+	       T6 = iio[-WS(ios, 6)];
+	       Tx = rio[WS(ios, 6)];
+	       {
+		    E T7, T8, Tu, Tv;
+		    T7 = iio[-WS(ios, 10)];
+		    T8 = rio[WS(ios, 2)];
+		    T9 = T7 + T8;
+		    T1j = KP866025403 * (T7 - T8);
+		    Tu = rio[WS(ios, 10)];
+		    Tv = iio[-WS(ios, 2)];
+		    Tw = Tu - Tv;
+		    T14 = KP866025403 * (Tu + Tv);
+	       }
+	       Ta = T6 + T9;
+	       Ty = Tw + Tx;
+	       T13 = FNMS(KP500000000, T9, T6);
+	       T15 = T13 + T14;
+	       T1N = T13 - T14;
+	       T1k = FMS(KP500000000, Tw, Tx);
+	       T1l = T1j + T1k;
+	       T1V = T1k - T1j;
+	  }
+	  {
+	       E Tc, TD, Tf, T17, TG, T1o, T18, T1n;
+	       Tc = rio[WS(ios, 3)];
+	       TD = iio[-WS(ios, 3)];
+	       {
+		    E Td, Te, TE, TF;
+		    Td = iio[-WS(ios, 7)];
+		    Te = iio[-WS(ios, 11)];
+		    Tf = Td + Te;
+		    T17 = KP866025403 * (Td - Te);
+		    TE = rio[WS(ios, 7)];
+		    TF = rio[WS(ios, 11)];
+		    TG = TE + TF;
+		    T1o = KP866025403 * (TE - TF);
+	       }
+	       Tg = Tc + Tf;
+	       TH = TD - TG;
+	       T18 = FMA(KP500000000, TG, TD);
+	       T19 = T17 + T18;
+	       T1X = T18 - T17;
+	       T1n = FNMS(KP500000000, Tf, Tc);
+	       T1p = T1n + T1o;
+	       T1P = T1n - T1o;
+	  }
+	  {
+	       E Tb, Tm, TU, TW, TX, TY, TT, TV;
+	       Tb = T5 + Ta;
+	       Tm = Tg + Tl;
+	       TU = Tb - Tm;
+	       TW = Tt - Ty;
+	       TX = TH + TM;
+	       TY = TW - TX;
+	       rio[0] = Tb + Tm;
+	       iio[-WS(ios, 11)] = TW + TX;
+	       TT = W[10];
+	       TV = W[11];
+	       rio[WS(ios, 6)] = FNMS(TV, TY, TT * TU);
+	       iio[-WS(ios, 5)] = FMA(TV, TU, TT * TY);
+	  }
+	  {
+	       E T28, T2g, T2c, T2e;
+	       {
+		    E T26, T27, T2a, T2b;
+		    T26 = T1M - T1N;
+		    T27 = T1X + T1Y;
+		    T28 = T26 - T27;
+		    T2g = T26 + T27;
+		    T2a = T1U - T1V;
+		    T2b = T1P - T1Q;
+		    T2c = T2a + T2b;
+		    T2e = T2a - T2b;
+	       }
+	       {
+		    E T25, T29, T2d, T2f;
+		    T25 = W[8];
+		    T29 = W[9];
+		    rio[WS(ios, 5)] = FNMS(T29, T2c, T25 * T28);
+		    iio[-WS(ios, 6)] = FMA(T25, T2c, T29 * T28);
+		    T2d = W[20];
+		    T2f = W[21];
+		    iio[0] = FMA(T2d, T2e, T2f * T2g);
+		    rio[WS(ios, 11)] = FNMS(T2f, T2e, T2d * T2g);
+	       }
+	  }
+	  {
+	       E TA, TS, TO, TQ;
+	       {
+		    E To, Tz, TC, TN;
+		    To = Tg - Tl;
+		    Tz = Tt + Ty;
+		    TA = To + Tz;
+		    TS = Tz - To;
+		    TC = T5 - Ta;
+		    TN = TH - TM;
+		    TO = TC - TN;
+		    TQ = TC + TN;
+	       }
+	       {
+		    E Tn, TB, TP, TR;
+		    Tn = W[16];
+		    TB = W[17];
+		    iio[-WS(ios, 2)] = FMA(Tn, TA, TB * TO);
+		    rio[WS(ios, 9)] = FNMS(TB, TA, Tn * TO);
+		    TP = W[4];
+		    TR = W[5];
+		    rio[WS(ios, 3)] = FNMS(TR, TS, TP * TQ);
+		    iio[-WS(ios, 8)] = FMA(TP, TS, TR * TQ);
+	       }
+	  }
+	  {
+	       E T1S, T22, T20, T24;
+	       {
+		    E T1O, T1R, T1W, T1Z;
+		    T1O = T1M + T1N;
+		    T1R = T1P + T1Q;
+		    T1S = T1O - T1R;
+		    T22 = T1O + T1R;
+		    T1W = T1U + T1V;
+		    T1Z = T1X - T1Y;
+		    T20 = T1W - T1Z;
+		    T24 = T1W + T1Z;
+	       }
+	       {
+		    E T1L, T1T, T21, T23;
+		    T1L = W[2];
+		    T1T = W[3];
+		    rio[WS(ios, 2)] = FNMS(T1T, T20, T1L * T1S);
+		    iio[-WS(ios, 9)] = FMA(T1T, T1S, T1L * T20);
+		    T21 = W[14];
+		    T23 = W[15];
+		    rio[WS(ios, 8)] = FNMS(T23, T24, T21 * T22);
+		    iio[-WS(ios, 3)] = FMA(T23, T22, T21 * T24);
+	       }
+	  }
+	  {
+	       E T1C, T1I, T1G, T1K;
+	       {
+		    E T1A, T1B, T1E, T1F;
+		    T1A = T12 + T15;
+		    T1B = T1p + T1s;
+		    T1C = T1A - T1B;
+		    T1I = T1A + T1B;
+		    T1E = T1i + T1l;
+		    T1F = T19 + T1c;
+		    T1G = T1E - T1F;
+		    T1K = T1E + T1F;
+	       }
+	       {
+		    E T1z, T1D, T1H, T1J;
+		    T1z = W[18];
+		    T1D = W[19];
+		    rio[WS(ios, 10)] = FNMS(T1D, T1G, T1z * T1C);
+		    iio[-WS(ios, 1)] = FMA(T1D, T1C, T1z * T1G);
+		    T1H = W[6];
+		    T1J = W[7];
+		    rio[WS(ios, 4)] = FNMS(T1J, T1K, T1H * T1I);
+		    iio[-WS(ios, 7)] = FMA(T1J, T1I, T1H * T1K);
+	       }
+	  }
+	  {
+	       E T1e, T1y, T1u, T1w;
+	       {
+		    E T16, T1d, T1m, T1t;
+		    T16 = T12 - T15;
+		    T1d = T19 - T1c;
+		    T1e = T16 - T1d;
+		    T1y = T16 + T1d;
+		    T1m = T1i - T1l;
+		    T1t = T1p - T1s;
+		    T1u = T1m + T1t;
+		    T1w = T1m - T1t;
+	       }
+	       {
+		    E TZ, T1f, T1v, T1x;
+		    TZ = W[0];
+		    T1f = W[1];
+		    rio[WS(ios, 1)] = FNMS(T1f, T1u, TZ * T1e);
+		    iio[-WS(ios, 10)] = FMA(TZ, T1u, T1f * T1e);
+		    T1v = W[12];
+		    T1x = W[13];
+		    iio[-WS(ios, 4)] = FMA(T1v, T1w, T1x * T1y);
+		    rio[WS(ios, 7)] = FNMS(T1x, T1w, T1v * T1y);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 12},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 12, "hb_12", twinstr, {88, 30, 30, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_12) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_12, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_15.c b/src/fftw3/rdft/codelets/hc2r/hb_15.c
new file mode 100644
index 0000000..8531f81
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_15.c
@@ -0,0 +1,404 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:36 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 15 -dif -name hb_15 -include hb.h */
+
+/*
+ * This function contains 184 FP additions, 112 FP multiplications,
+ * (or, 128 additions, 56 multiplications, 56 fused multiply/add),
+ * 75 stack variables, and 60 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_15.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_15.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_15.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_15(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 28) {
+	  E T5, T2N, TV, T25, T1v, T2o, T2W, T38, T37, T2X, T2Q, T2T, T2U, Tg, Tr;
+	  E Ts, Ty, TD, TE, T2c, T2d, T2m, TJ, TO, TP, T1Y, T1Z, T20, T1e, T1j;
+	  E T1p, T13, T18, T1o, T29, T2a, T2l, T21, T22, T23;
+	  {
+	       E T1, T1r, T4, T1u, TU, T1s, TR, T1t;
+	       T1 = rio[0];
+	       T1r = iio[0];
+	       {
+		    E T2, T3, TS, TT;
+		    T2 = rio[WS(ios, 5)];
+		    T3 = iio[-WS(ios, 10)];
+		    T4 = T2 + T3;
+		    T1u = KP866025403 * (T2 - T3);
+		    TS = rio[WS(ios, 10)];
+		    TT = iio[-WS(ios, 5)];
+		    TU = KP866025403 * (TS + TT);
+		    T1s = TS - TT;
+	       }
+	       T5 = T1 + T4;
+	       T2N = T1r - T1s;
+	       TR = FNMS(KP500000000, T4, T1);
+	       TV = TR + TU;
+	       T25 = TR - TU;
+	       T1t = FMA(KP500000000, T1s, T1r);
+	       T1v = T1t - T1u;
+	       T2o = T1u + T1t;
+	  }
+	  {
+	       E Ta, T12, Tu, Tx, T2O, T11, Tf, T14, Tz, TC, T2P, T17, Tl, T1d, TF;
+	       E TI, T2R, T1c, Tq, T1i, TK, TN, T2S, T1h;
+	       {
+		    E T6, T7, T8, T9;
+		    T6 = rio[WS(ios, 3)];
+		    T7 = iio[-WS(ios, 8)];
+		    T8 = iio[-WS(ios, 13)];
+		    T9 = T7 + T8;
+		    Ta = T6 + T9;
+		    T12 = KP866025403 * (T7 - T8);
+		    Tu = FNMS(KP500000000, T9, T6);
+	       }
+	       {
+		    E TZ, Tv, Tw, T10;
+		    TZ = iio[-WS(ios, 3)];
+		    Tv = rio[WS(ios, 8)];
+		    Tw = rio[WS(ios, 13)];
+		    T10 = Tv + Tw;
+		    Tx = KP866025403 * (Tv - Tw);
+		    T2O = TZ - T10;
+		    T11 = FMA(KP500000000, T10, TZ);
+	       }
+	       {
+		    E Tb, Tc, Td, Te;
+		    Tb = iio[-WS(ios, 12)];
+		    Tc = rio[WS(ios, 2)];
+		    Td = rio[WS(ios, 7)];
+		    Te = Tc + Td;
+		    Tf = Tb + Te;
+		    T14 = KP866025403 * (Tc - Td);
+		    Tz = FNMS(KP500000000, Te, Tb);
+	       }
+	       {
+		    E T15, TA, TB, T16;
+		    T15 = rio[WS(ios, 12)];
+		    TA = iio[-WS(ios, 7)];
+		    TB = iio[-WS(ios, 2)];
+		    T16 = TB + TA;
+		    TC = KP866025403 * (TA - TB);
+		    T2P = T16 - T15;
+		    T17 = FMA(KP500000000, T16, T15);
+	       }
+	       {
+		    E Th, Ti, Tj, Tk;
+		    Th = rio[WS(ios, 6)];
+		    Ti = iio[-WS(ios, 11)];
+		    Tj = rio[WS(ios, 1)];
+		    Tk = Ti + Tj;
+		    Tl = Th + Tk;
+		    T1d = KP866025403 * (Ti - Tj);
+		    TF = FNMS(KP500000000, Tk, Th);
+	       }
+	       {
+		    E T1a, TG, TH, T1b;
+		    T1a = iio[-WS(ios, 6)];
+		    TG = rio[WS(ios, 11)];
+		    TH = iio[-WS(ios, 1)];
+		    T1b = TG - TH;
+		    TI = KP866025403 * (TG + TH);
+		    T2R = T1a - T1b;
+		    T1c = FMA(KP500000000, T1b, T1a);
+	       }
+	       {
+		    E Tm, Tn, To, Tp;
+		    Tm = iio[-WS(ios, 9)];
+		    Tn = iio[-WS(ios, 14)];
+		    To = rio[WS(ios, 4)];
+		    Tp = Tn + To;
+		    Tq = Tm + Tp;
+		    T1i = KP866025403 * (Tn - To);
+		    TK = FNMS(KP500000000, Tp, Tm);
+	       }
+	       {
+		    E T1g, TL, TM, T1f;
+		    T1g = rio[WS(ios, 9)];
+		    TL = rio[WS(ios, 14)];
+		    TM = iio[-WS(ios, 4)];
+		    T1f = TL - TM;
+		    TN = KP866025403 * (TL + TM);
+		    T2S = T1f + T1g;
+		    T1h = FMS(KP500000000, T1f, T1g);
+	       }
+	       T2W = Ta - Tf;
+	       T38 = T2R + T2S;
+	       T37 = T2O - T2P;
+	       T2X = Tl - Tq;
+	       T2Q = T2O + T2P;
+	       T2T = T2R - T2S;
+	       T2U = T2Q + T2T;
+	       Tg = Ta + Tf;
+	       Tr = Tl + Tq;
+	       Ts = Tg + Tr;
+	       Ty = Tu - Tx;
+	       TD = Tz - TC;
+	       TE = Ty + TD;
+	       T2c = T1d + T1c;
+	       T2d = T1i + T1h;
+	       T2m = T2c + T2d;
+	       TJ = TF - TI;
+	       TO = TK - TN;
+	       TP = TJ + TO;
+	       T1Y = Tu + Tx;
+	       T1Z = Tz + TC;
+	       T20 = T1Y + T1Z;
+	       T1e = T1c - T1d;
+	       T1j = T1h - T1i;
+	       T1p = T1e + T1j;
+	       T13 = T11 - T12;
+	       T18 = T14 + T17;
+	       T1o = T13 - T18;
+	       T29 = T12 + T11;
+	       T2a = T14 - T17;
+	       T2l = T29 + T2a;
+	       T21 = TF + TI;
+	       T22 = TK + TN;
+	       T23 = T21 + T22;
+	  }
+	  rio[0] = T5 + Ts;
+	  {
+	       E T1l, T1J, T1B, T1M, TY, T1U, T1I, T1y, T1W, T1N, T1T, T1V;
+	       {
+		    E T19, T1k, T1z, T1A;
+		    T19 = T13 + T18;
+		    T1k = T1e - T1j;
+		    T1l = FMA(KP951056516, T19, KP587785252 * T1k);
+		    T1J = FNMS(KP951056516, T1k, KP587785252 * T19);
+		    T1z = Ty - TD;
+		    T1A = TJ - TO;
+		    T1B = FMA(KP951056516, T1z, KP587785252 * T1A);
+		    T1M = FNMS(KP951056516, T1A, KP587785252 * T1z);
+	       }
+	       {
+		    E TQ, TW, TX, T1q, T1w, T1x;
+		    TQ = KP559016994 * (TE - TP);
+		    TW = TE + TP;
+		    TX = FNMS(KP250000000, TW, TV);
+		    TY = TQ + TX;
+		    T1U = TV + TW;
+		    T1I = TX - TQ;
+		    T1q = KP559016994 * (T1o - T1p);
+		    T1w = T1o + T1p;
+		    T1x = FNMS(KP250000000, T1w, T1v);
+		    T1y = T1q + T1x;
+		    T1W = T1v + T1w;
+		    T1N = T1x - T1q;
+	       }
+	       T1T = W[8];
+	       T1V = W[9];
+	       rio[WS(ios, 5)] = FNMS(T1V, T1W, T1T * T1U);
+	       iio[-WS(ios, 9)] = FMA(T1V, T1U, T1T * T1W);
+	       {
+		    E T1Q, T1S, T1P, T1R;
+		    T1Q = T1I + T1J;
+		    T1S = T1N - T1M;
+		    T1P = W[14];
+		    T1R = W[15];
+		    rio[WS(ios, 8)] = FNMS(T1R, T1S, T1P * T1Q);
+		    iio[-WS(ios, 6)] = FMA(T1R, T1Q, T1P * T1S);
+	       }
+	       {
+		    E T1m, T1C, Tt, T1n;
+		    T1m = TY + T1l;
+		    T1C = T1y - T1B;
+		    Tt = W[26];
+		    T1n = W[27];
+		    rio[WS(ios, 14)] = FNMS(T1n, T1C, Tt * T1m);
+		    iio[0] = FMA(T1n, T1m, Tt * T1C);
+	       }
+	       {
+		    E T1E, T1G, T1D, T1F;
+		    T1E = TY - T1l;
+		    T1G = T1B + T1y;
+		    T1D = W[20];
+		    T1F = W[21];
+		    rio[WS(ios, 11)] = FNMS(T1F, T1G, T1D * T1E);
+		    iio[-WS(ios, 3)] = FMA(T1F, T1E, T1D * T1G);
+	       }
+	       {
+		    E T1K, T1O, T1H, T1L;
+		    T1K = T1I - T1J;
+		    T1O = T1M + T1N;
+		    T1H = W[2];
+		    T1L = W[3];
+		    rio[WS(ios, 2)] = FNMS(T1L, T1O, T1H * T1K);
+		    iio[-WS(ios, 12)] = FMA(T1L, T1K, T1H * T1O);
+	       }
+	  }
+	  iio[-WS(ios, 14)] = T2N + T2U;
+	  {
+	       E T2Y, T39, T3k, T3h, T36, T3g, T31, T3l;
+	       T2Y = FNMS(KP951056516, T2X, KP587785252 * T2W);
+	       T39 = FNMS(KP951056516, T38, KP587785252 * T37);
+	       T3k = FMA(KP951056516, T2W, KP587785252 * T2X);
+	       T3h = FMA(KP951056516, T37, KP587785252 * T38);
+	       {
+		    E T34, T35, T2Z, T30;
+		    T34 = FNMS(KP250000000, Ts, T5);
+		    T35 = KP559016994 * (Tg - Tr);
+		    T36 = T34 - T35;
+		    T3g = T35 + T34;
+		    T2Z = FNMS(KP250000000, T2U, T2N);
+		    T30 = KP559016994 * (T2Q - T2T);
+		    T31 = T2Z - T30;
+		    T3l = T30 + T2Z;
+	       }
+	       {
+		    E T32, T3a, T2V, T33;
+		    T32 = T2Y + T31;
+		    T3a = T36 - T39;
+		    T2V = W[22];
+		    T33 = W[23];
+		    iio[-WS(ios, 2)] = FMA(T2V, T32, T33 * T3a);
+		    rio[WS(ios, 12)] = FNMS(T33, T32, T2V * T3a);
+	       }
+	       {
+		    E T3o, T3q, T3n, T3p;
+		    T3o = T3l - T3k;
+		    T3q = T3g + T3h;
+		    T3n = W[16];
+		    T3p = W[17];
+		    iio[-WS(ios, 5)] = FMA(T3n, T3o, T3p * T3q);
+		    rio[WS(ios, 9)] = FNMS(T3p, T3o, T3n * T3q);
+	       }
+	       {
+		    E T3c, T3e, T3b, T3d;
+		    T3c = T36 + T39;
+		    T3e = T31 - T2Y;
+		    T3b = W[4];
+		    T3d = W[5];
+		    rio[WS(ios, 3)] = FNMS(T3d, T3e, T3b * T3c);
+		    iio[-WS(ios, 11)] = FMA(T3b, T3e, T3d * T3c);
+	       }
+	       {
+		    E T3i, T3m, T3f, T3j;
+		    T3i = T3g - T3h;
+		    T3m = T3k + T3l;
+		    T3f = W[10];
+		    T3j = W[11];
+		    rio[WS(ios, 6)] = FNMS(T3j, T3m, T3f * T3i);
+		    iio[-WS(ios, 8)] = FMA(T3f, T3m, T3j * T3i);
+	       }
+	  }
+	  {
+	       E T2f, T2z, T2k, T2D, T28, T2K, T2y, T2r, T2M, T2C, T2J, T2L;
+	       {
+		    E T2b, T2e, T2i, T2j;
+		    T2b = T29 - T2a;
+		    T2e = T2c - T2d;
+		    T2f = FMA(KP951056516, T2b, KP587785252 * T2e);
+		    T2z = FNMS(KP951056516, T2e, KP587785252 * T2b);
+		    T2i = T1Y - T1Z;
+		    T2j = T21 - T22;
+		    T2k = FMA(KP951056516, T2i, KP587785252 * T2j);
+		    T2D = FNMS(KP951056516, T2j, KP587785252 * T2i);
+	       }
+	       {
+		    E T24, T26, T27, T2n, T2p, T2q;
+		    T24 = KP559016994 * (T20 - T23);
+		    T26 = T20 + T23;
+		    T27 = FNMS(KP250000000, T26, T25);
+		    T28 = T24 + T27;
+		    T2K = T25 + T26;
+		    T2y = T27 - T24;
+		    T2n = KP559016994 * (T2l - T2m);
+		    T2p = T2l + T2m;
+		    T2q = FNMS(KP250000000, T2p, T2o);
+		    T2r = T2n + T2q;
+		    T2M = T2o + T2p;
+		    T2C = T2q - T2n;
+	       }
+	       T2J = W[18];
+	       T2L = W[19];
+	       rio[WS(ios, 10)] = FNMS(T2L, T2M, T2J * T2K);
+	       iio[-WS(ios, 4)] = FMA(T2L, T2K, T2J * T2M);
+	       {
+		    E T2u, T2w, T2t, T2v;
+		    T2u = T28 + T2f;
+		    T2w = T2r - T2k;
+		    T2t = W[6];
+		    T2v = W[7];
+		    rio[WS(ios, 4)] = FNMS(T2v, T2w, T2t * T2u);
+		    iio[-WS(ios, 10)] = FMA(T2v, T2u, T2t * T2w);
+	       }
+	       {
+		    E T2g, T2s, T1X, T2h;
+		    T2g = T28 - T2f;
+		    T2s = T2k + T2r;
+		    T1X = W[0];
+		    T2h = W[1];
+		    rio[WS(ios, 1)] = FNMS(T2h, T2s, T1X * T2g);
+		    iio[-WS(ios, 13)] = FMA(T2h, T2g, T1X * T2s);
+	       }
+	       {
+		    E T2A, T2E, T2x, T2B;
+		    T2A = T2y + T2z;
+		    T2E = T2C - T2D;
+		    T2x = W[24];
+		    T2B = W[25];
+		    rio[WS(ios, 13)] = FNMS(T2B, T2E, T2x * T2A);
+		    iio[-WS(ios, 1)] = FMA(T2B, T2A, T2x * T2E);
+	       }
+	       {
+		    E T2G, T2I, T2F, T2H;
+		    T2G = T2y - T2z;
+		    T2I = T2D + T2C;
+		    T2F = W[12];
+		    T2H = W[13];
+		    rio[WS(ios, 7)] = FNMS(T2H, T2I, T2F * T2G);
+		    iio[-WS(ios, 7)] = FMA(T2H, T2G, T2F * T2I);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 15},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 15, "hb_15", twinstr, {128, 56, 56, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_15) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_15, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_16.c b/src/fftw3/rdft/codelets/hc2r/hb_16.c
new file mode 100644
index 0000000..7fffeb9
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_16.c
@@ -0,0 +1,412 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:37 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 16 -dif -name hb_16 -include hb.h */
+
+/*
+ * This function contains 174 FP additions, 84 FP multiplications,
+ * (or, 136 additions, 46 multiplications, 38 fused multiply/add),
+ * 50 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_16.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_16.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_16.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_16(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 30) {
+	  E T7, T2K, T30, Tw, T1a, T2e, T2k, T1B, Te, TD, T1C, T13, T2n, T2Z, T2b;
+	  E T2L, Tm, T1v, TN, T10, T1W, T2p, T2P, T2W, Tt, T1w, TW, T11, T23, T2q;
+	  E T2S, T2X;
+	  {
+	       E T3, T2c, T16, T2j, T6, T2i, T19, T2d;
+	       {
+		    E T1, T2, T14, T15;
+		    T1 = rio[0];
+		    T2 = iio[-WS(ios, 8)];
+		    T3 = T1 + T2;
+		    T2c = T1 - T2;
+		    T14 = iio[0];
+		    T15 = rio[WS(ios, 8)];
+		    T16 = T14 - T15;
+		    T2j = T14 + T15;
+	       }
+	       {
+		    E T4, T5, T17, T18;
+		    T4 = rio[WS(ios, 4)];
+		    T5 = iio[-WS(ios, 12)];
+		    T6 = T4 + T5;
+		    T2i = T4 - T5;
+		    T17 = iio[-WS(ios, 4)];
+		    T18 = rio[WS(ios, 12)];
+		    T19 = T17 - T18;
+		    T2d = T17 + T18;
+	       }
+	       T7 = T3 + T6;
+	       T2K = T2j - T2i;
+	       T30 = T2c + T2d;
+	       Tw = T3 - T6;
+	       T1a = T16 - T19;
+	       T2e = T2c - T2d;
+	       T2k = T2i + T2j;
+	       T1B = T16 + T19;
+	  }
+	  {
+	       E Ta, T29, Tz, T28, Td, T25, TC, T26;
+	       {
+		    E T8, T9, Tx, Ty;
+		    T8 = rio[WS(ios, 2)];
+		    T9 = iio[-WS(ios, 10)];
+		    Ta = T8 + T9;
+		    T29 = T8 - T9;
+		    Tx = iio[-WS(ios, 2)];
+		    Ty = rio[WS(ios, 10)];
+		    Tz = Tx - Ty;
+		    T28 = Tx + Ty;
+	       }
+	       {
+		    E Tb, Tc, TA, TB;
+		    Tb = iio[-WS(ios, 14)];
+		    Tc = rio[WS(ios, 6)];
+		    Td = Tb + Tc;
+		    T25 = Tb - Tc;
+		    TA = iio[-WS(ios, 6)];
+		    TB = rio[WS(ios, 14)];
+		    TC = TA - TB;
+		    T26 = TB + TA;
+	       }
+	       Te = Ta + Td;
+	       TD = Tz - TC;
+	       T1C = Tz + TC;
+	       T13 = Td - Ta;
+	       {
+		    E T2l, T2m, T27, T2a;
+		    T2l = T29 + T28;
+		    T2m = T25 + T26;
+		    T2n = KP707106781 * (T2l - T2m);
+		    T2Z = KP707106781 * (T2l + T2m);
+		    T27 = T25 - T26;
+		    T2a = T28 - T29;
+		    T2b = KP707106781 * (T27 - T2a);
+		    T2L = KP707106781 * (T2a + T27);
+	       }
+	  }
+	  {
+	       E Ti, T1Q, TI, T1U, Tl, T1T, TL, T1R, TF, TM;
+	       {
+		    E Tg, Th, TG, TH;
+		    Tg = rio[WS(ios, 1)];
+		    Th = iio[-WS(ios, 9)];
+		    Ti = Tg + Th;
+		    T1Q = Tg - Th;
+		    TG = iio[-WS(ios, 1)];
+		    TH = rio[WS(ios, 9)];
+		    TI = TG - TH;
+		    T1U = TG + TH;
+	       }
+	       {
+		    E Tj, Tk, TJ, TK;
+		    Tj = rio[WS(ios, 5)];
+		    Tk = iio[-WS(ios, 13)];
+		    Tl = Tj + Tk;
+		    T1T = Tj - Tk;
+		    TJ = iio[-WS(ios, 5)];
+		    TK = rio[WS(ios, 13)];
+		    TL = TJ - TK;
+		    T1R = TJ + TK;
+	       }
+	       Tm = Ti + Tl;
+	       T1v = TI + TL;
+	       TF = Ti - Tl;
+	       TM = TI - TL;
+	       TN = TF + TM;
+	       T10 = TM - TF;
+	       {
+		    E T1S, T1V, T2N, T2O;
+		    T1S = T1Q - T1R;
+		    T1V = T1T + T1U;
+		    T1W = FNMS(KP382683432, T1V, KP923879532 * T1S);
+		    T2p = FMA(KP923879532, T1V, KP382683432 * T1S);
+		    T2N = T1U - T1T;
+		    T2O = T1Q + T1R;
+		    T2P = FNMS(KP382683432, T2O, KP923879532 * T2N);
+		    T2W = FMA(KP382683432, T2N, KP923879532 * T2O);
+	       }
+	  }
+	  {
+	       E Tp, T1X, TR, T21, Ts, T20, TU, T1Y, TO, TV;
+	       {
+		    E Tn, To, TP, TQ;
+		    Tn = iio[-WS(ios, 15)];
+		    To = rio[WS(ios, 7)];
+		    Tp = Tn + To;
+		    T1X = Tn - To;
+		    TP = iio[-WS(ios, 7)];
+		    TQ = rio[WS(ios, 15)];
+		    TR = TP - TQ;
+		    T21 = TQ + TP;
+	       }
+	       {
+		    E Tq, Tr, TS, TT;
+		    Tq = rio[WS(ios, 3)];
+		    Tr = iio[-WS(ios, 11)];
+		    Ts = Tq + Tr;
+		    T20 = Tq - Tr;
+		    TS = iio[-WS(ios, 3)];
+		    TT = rio[WS(ios, 11)];
+		    TU = TS - TT;
+		    T1Y = TS + TT;
+	       }
+	       Tt = Tp + Ts;
+	       T1w = TU + TR;
+	       TO = Tp - Ts;
+	       TV = TR - TU;
+	       TW = TO - TV;
+	       T11 = TO + TV;
+	       {
+		    E T1Z, T22, T2Q, T2R;
+		    T1Z = T1X - T1Y;
+		    T22 = T20 - T21;
+		    T23 = FMA(KP923879532, T1Z, KP382683432 * T22);
+		    T2q = FNMS(KP382683432, T1Z, KP923879532 * T22);
+		    T2Q = T1X + T1Y;
+		    T2R = T20 + T21;
+		    T2S = FNMS(KP923879532, T2R, KP382683432 * T2Q);
+		    T2X = FMA(KP923879532, T2Q, KP382683432 * T2R);
+	       }
+	  }
+	  {
+	       E Tf, Tu, T1K, T1M, T1N, T1O, T1J, T1L;
+	       Tf = T7 + Te;
+	       Tu = Tm + Tt;
+	       T1K = Tf - Tu;
+	       T1M = T1C + T1B;
+	       T1N = T1v + T1w;
+	       T1O = T1M - T1N;
+	       rio[0] = Tf + Tu;
+	       iio[-WS(ios, 15)] = T1N + T1M;
+	       T1J = W[14];
+	       T1L = W[15];
+	       rio[WS(ios, 8)] = FNMS(T1L, T1O, T1J * T1K);
+	       iio[-WS(ios, 7)] = FMA(T1L, T1K, T1J * T1O);
+	  }
+	  {
+	       E T2U, T36, T32, T34;
+	       {
+		    E T2M, T2T, T2Y, T31;
+		    T2M = T2K + T2L;
+		    T2T = T2P + T2S;
+		    T2U = T2M + T2T;
+		    T36 = T2M - T2T;
+		    T2Y = T2W + T2X;
+		    T31 = T2Z + T30;
+		    T32 = T2Y + T31;
+		    T34 = T31 - T2Y;
+	       }
+	       {
+		    E T2J, T2V, T33, T35;
+		    T2J = W[28];
+		    T2V = W[29];
+		    iio[0] = FMA(T2J, T2U, T2V * T32);
+		    rio[WS(ios, 15)] = FNMS(T2V, T2U, T2J * T32);
+		    T33 = W[12];
+		    T35 = W[13];
+		    rio[WS(ios, 7)] = FNMS(T35, T36, T33 * T34);
+		    iio[-WS(ios, 8)] = FMA(T33, T36, T35 * T34);
+	       }
+	  }
+	  {
+	       E TY, T1e, T1c, T1g;
+	       {
+		    E TE, TX, T12, T1b;
+		    TE = Tw + TD;
+		    TX = KP707106781 * (TN + TW);
+		    TY = TE + TX;
+		    T1e = TE - TX;
+		    T12 = KP707106781 * (T10 + T11);
+		    T1b = T13 + T1a;
+		    T1c = T12 + T1b;
+		    T1g = T1b - T12;
+	       }
+	       {
+		    E Tv, TZ, T1d, T1f;
+		    Tv = W[26];
+		    TZ = W[27];
+		    rio[WS(ios, 14)] = FNMS(TZ, T1c, Tv * TY);
+		    iio[-WS(ios, 1)] = FMA(TZ, TY, Tv * T1c);
+		    T1d = W[10];
+		    T1f = W[11];
+		    rio[WS(ios, 6)] = FNMS(T1f, T1g, T1d * T1e);
+		    iio[-WS(ios, 9)] = FMA(T1f, T1e, T1d * T1g);
+	       }
+	  }
+	  {
+	       E T2g, T2w, T2s, T2u;
+	       {
+		    E T24, T2f, T2o, T2r;
+		    T24 = T1W + T23;
+		    T2f = T2b + T2e;
+		    T2g = T24 + T2f;
+		    T2w = T2f - T24;
+		    T2o = T2k + T2n;
+		    T2r = T2p + T2q;
+		    T2s = T2o + T2r;
+		    T2u = T2o - T2r;
+	       }
+	       {
+		    E T1P, T2h, T2t, T2v;
+		    T1P = W[0];
+		    T2h = W[1];
+		    rio[WS(ios, 1)] = FNMS(T2h, T2s, T1P * T2g);
+		    iio[-WS(ios, 14)] = FMA(T1P, T2s, T2h * T2g);
+		    T2t = W[16];
+		    T2v = W[17];
+		    iio[-WS(ios, 6)] = FMA(T2t, T2u, T2v * T2w);
+		    rio[WS(ios, 9)] = FNMS(T2v, T2u, T2t * T2w);
+	       }
+	  }
+	  {
+	       E T1k, T1q, T1o, T1s;
+	       {
+		    E T1i, T1j, T1m, T1n;
+		    T1i = Tw - TD;
+		    T1j = KP707106781 * (T11 - T10);
+		    T1k = T1i + T1j;
+		    T1q = T1i - T1j;
+		    T1m = KP707106781 * (TN - TW);
+		    T1n = T1a - T13;
+		    T1o = T1m + T1n;
+		    T1s = T1n - T1m;
+	       }
+	       {
+		    E T1h, T1l, T1p, T1r;
+		    T1h = W[2];
+		    T1l = W[3];
+		    rio[WS(ios, 2)] = FNMS(T1l, T1o, T1h * T1k);
+		    iio[-WS(ios, 13)] = FMA(T1l, T1k, T1h * T1o);
+		    T1p = W[18];
+		    T1r = W[19];
+		    rio[WS(ios, 10)] = FNMS(T1r, T1s, T1p * T1q);
+		    iio[-WS(ios, 5)] = FMA(T1r, T1q, T1p * T1s);
+	       }
+	  }
+	  {
+	       E T2A, T2I, T2E, T2G;
+	       {
+		    E T2y, T2z, T2C, T2D;
+		    T2y = T2k - T2n;
+		    T2z = T23 - T1W;
+		    T2A = T2y + T2z;
+		    T2I = T2y - T2z;
+		    T2C = T2p - T2q;
+		    T2D = T2e - T2b;
+		    T2E = T2C + T2D;
+		    T2G = T2D - T2C;
+	       }
+	       {
+		    E T2x, T2B, T2F, T2H;
+		    T2x = W[24];
+		    T2B = W[25];
+		    iio[-WS(ios, 2)] = FMA(T2x, T2A, T2B * T2E);
+		    rio[WS(ios, 13)] = FNMS(T2B, T2A, T2x * T2E);
+		    T2F = W[8];
+		    T2H = W[9];
+		    rio[WS(ios, 5)] = FNMS(T2H, T2I, T2F * T2G);
+		    iio[-WS(ios, 10)] = FMA(T2F, T2I, T2H * T2G);
+	       }
+	  }
+	  {
+	       E T1y, T1G, T1E, T1I;
+	       {
+		    E T1u, T1x, T1A, T1D;
+		    T1u = T7 - Te;
+		    T1x = T1v - T1w;
+		    T1y = T1u + T1x;
+		    T1G = T1u - T1x;
+		    T1A = Tt - Tm;
+		    T1D = T1B - T1C;
+		    T1E = T1A + T1D;
+		    T1I = T1D - T1A;
+	       }
+	       {
+		    E T1t, T1z, T1F, T1H;
+		    T1t = W[22];
+		    T1z = W[23];
+		    rio[WS(ios, 12)] = FNMS(T1z, T1E, T1t * T1y);
+		    iio[-WS(ios, 3)] = FMA(T1z, T1y, T1t * T1E);
+		    T1F = W[6];
+		    T1H = W[7];
+		    rio[WS(ios, 4)] = FNMS(T1H, T1I, T1F * T1G);
+		    iio[-WS(ios, 11)] = FMA(T1H, T1G, T1F * T1I);
+	       }
+	  }
+	  {
+	       E T3a, T3i, T3e, T3g;
+	       {
+		    E T38, T39, T3c, T3d;
+		    T38 = T2S - T2P;
+		    T39 = T30 - T2Z;
+		    T3a = T38 + T39;
+		    T3i = T39 - T38;
+		    T3c = T2K - T2L;
+		    T3d = T2W - T2X;
+		    T3e = T3c + T3d;
+		    T3g = T3c - T3d;
+	       }
+	       {
+		    E T37, T3b, T3f, T3h;
+		    T37 = W[4];
+		    T3b = W[5];
+		    rio[WS(ios, 3)] = FNMS(T3b, T3e, T37 * T3a);
+		    iio[-WS(ios, 12)] = FMA(T37, T3e, T3b * T3a);
+		    T3f = W[20];
+		    T3h = W[21];
+		    iio[-WS(ios, 4)] = FMA(T3f, T3g, T3h * T3i);
+		    rio[WS(ios, 11)] = FNMS(T3h, T3g, T3f * T3i);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 16},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 16, "hb_16", twinstr, {136, 46, 38, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_16) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_16, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_2.c b/src/fftw3/rdft/codelets/hc2r/hb_2.c
new file mode 100644
index 0000000..91740d2
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_2.c
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:22 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 2 -dif -name hb_2 -include hb.h */
+
+/*
+ * This function contains 6 FP additions, 4 FP multiplications,
+ * (or, 4 additions, 2 multiplications, 2 fused multiply/add),
+ * 9 stack variables, and 8 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_2.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_2.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_2.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_2(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 2) {
+	  E T1, T2, T6, T3, T4, T8, T5, T7;
+	  T1 = rio[0];
+	  T2 = iio[-WS(ios, 1)];
+	  T6 = T1 - T2;
+	  T3 = iio[0];
+	  T4 = rio[WS(ios, 1)];
+	  T8 = T3 + T4;
+	  rio[0] = T1 + T2;
+	  iio[-WS(ios, 1)] = T3 - T4;
+	  T5 = W[0];
+	  T7 = W[1];
+	  rio[WS(ios, 1)] = FNMS(T7, T8, T5 * T6);
+	  iio[0] = FMA(T7, T6, T5 * T8);
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 2},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 2, "hb_2", twinstr, {4, 2, 2, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_2) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_2, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_3.c b/src/fftw3/rdft/codelets/hc2r/hb_3.c
new file mode 100644
index 0000000..b2362cb
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_3.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:22 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 3 -dif -name hb_3 -include hb.h */
+
+/*
+ * This function contains 16 FP additions, 12 FP multiplications,
+ * (or, 10 additions, 6 multiplications, 6 fused multiply/add),
+ * 15 stack variables, and 12 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_3.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_3.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_3.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_3(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 4) {
+	  E T1, T4, Ta, Te, T5, T8, Tb, Tf;
+	  {
+	       E T2, T3, T6, T7;
+	       T1 = rio[0];
+	       T2 = rio[WS(ios, 1)];
+	       T3 = iio[-WS(ios, 2)];
+	       T4 = T2 + T3;
+	       Ta = FNMS(KP500000000, T4, T1);
+	       Te = KP866025403 * (T2 - T3);
+	       T5 = iio[0];
+	       T6 = rio[WS(ios, 2)];
+	       T7 = iio[-WS(ios, 1)];
+	       T8 = T6 - T7;
+	       Tb = KP866025403 * (T6 + T7);
+	       Tf = FMA(KP500000000, T8, T5);
+	  }
+	  rio[0] = T1 + T4;
+	  iio[-WS(ios, 2)] = T5 - T8;
+	  {
+	       E Ti, Tk, Th, Tj;
+	       Ti = Tf - Te;
+	       Tk = Ta + Tb;
+	       Th = W[2];
+	       Tj = W[3];
+	       iio[0] = FMA(Th, Ti, Tj * Tk);
+	       rio[WS(ios, 2)] = FNMS(Tj, Ti, Th * Tk);
+	  }
+	  {
+	       E Tc, Tg, T9, Td;
+	       Tc = Ta - Tb;
+	       Tg = Te + Tf;
+	       T9 = W[0];
+	       Td = W[1];
+	       rio[WS(ios, 1)] = FNMS(Td, Tg, T9 * Tc);
+	       iio[-WS(ios, 1)] = FMA(T9, Tg, Td * Tc);
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 3},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 3, "hb_3", twinstr, {10, 6, 6, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_3) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_3, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_32.c b/src/fftw3/rdft/codelets/hc2r/hb_32.c
new file mode 100644
index 0000000..f3358ab
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_32.c
@@ -0,0 +1,890 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:39 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 32 -dif -name hb_32 -include hb.h */
+
+/*
+ * This function contains 434 FP additions, 208 FP multiplications,
+ * (or, 340 additions, 114 multiplications, 94 fused multiply/add),
+ * 98 stack variables, and 128 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_32(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 62) {
+	  E T5n, T6y, T77, T5u, Tf, T3i, T5x, T76, T3G, T47, T1a, T2I, T5k, T6z, T2o;
+	  E T2Y, Tu, T3D, T6D, T73, T6G, T74, T1j, T2d, T1s, T2e, T55, T5z, T5c, T5A;
+	  E T3l, T48, TK, T3n, T6L, T7t, T6O, T7s, T1D, T2L, T1M, T2M, T4w, T62, T4D;
+	  E T61, T3q, T41, TZ, T3s, T6S, T7w, T6V, T7v, T1W, T2O, T25, T2P, T4P, T64;
+	  E T4W, T65, T3v, T42;
+	  {
+	       E T3, T5l, T2j, T5t, T6, T5s, T2m, T5m, Ta, T5i, T15, T5h, Td, T5e, T18;
+	       E T5f;
+	       {
+		    E T1, T2, T2h, T2i;
+		    T1 = rio[0];
+		    T2 = iio[-WS(ios, 16)];
+		    T3 = T1 + T2;
+		    T5l = T1 - T2;
+		    T2h = iio[0];
+		    T2i = rio[WS(ios, 16)];
+		    T2j = T2h - T2i;
+		    T5t = T2h + T2i;
+	       }
+	       {
+		    E T4, T5, T2k, T2l;
+		    T4 = rio[WS(ios, 8)];
+		    T5 = iio[-WS(ios, 24)];
+		    T6 = T4 + T5;
+		    T5s = T4 - T5;
+		    T2k = iio[-WS(ios, 8)];
+		    T2l = rio[WS(ios, 24)];
+		    T2m = T2k - T2l;
+		    T5m = T2k + T2l;
+	       }
+	       {
+		    E T8, T9, T13, T14;
+		    T8 = rio[WS(ios, 4)];
+		    T9 = iio[-WS(ios, 20)];
+		    Ta = T8 + T9;
+		    T5i = T8 - T9;
+		    T13 = iio[-WS(ios, 4)];
+		    T14 = rio[WS(ios, 20)];
+		    T15 = T13 - T14;
+		    T5h = T13 + T14;
+	       }
+	       {
+		    E Tb, Tc, T16, T17;
+		    Tb = iio[-WS(ios, 28)];
+		    Tc = rio[WS(ios, 12)];
+		    Td = Tb + Tc;
+		    T5e = Tb - Tc;
+		    T16 = iio[-WS(ios, 12)];
+		    T17 = rio[WS(ios, 28)];
+		    T18 = T16 - T17;
+		    T5f = T17 + T16;
+	       }
+	       {
+		    E T7, Te, T12, T19;
+		    T5n = T5l - T5m;
+		    T6y = T5t - T5s;
+		    T77 = T5l + T5m;
+		    T5u = T5s + T5t;
+		    T7 = T3 + T6;
+		    Te = Ta + Td;
+		    Tf = T7 + Te;
+		    T3i = T7 - Te;
+		    {
+			 E T5v, T5w, T3E, T3F;
+			 T5v = T5i + T5h;
+			 T5w = T5e + T5f;
+			 T5x = KP707106781 * (T5v - T5w);
+			 T76 = KP707106781 * (T5v + T5w);
+			 T3E = T2j + T2m;
+			 T3F = T15 + T18;
+			 T3G = T3E - T3F;
+			 T47 = T3F + T3E;
+		    }
+		    T12 = T3 - T6;
+		    T19 = T15 - T18;
+		    T1a = T12 + T19;
+		    T2I = T12 - T19;
+		    {
+			 E T5g, T5j, T2g, T2n;
+			 T5g = T5e - T5f;
+			 T5j = T5h - T5i;
+			 T5k = KP707106781 * (T5g - T5j);
+			 T6z = KP707106781 * (T5j + T5g);
+			 T2g = Td - Ta;
+			 T2n = T2j - T2m;
+			 T2o = T2g + T2n;
+			 T2Y = T2n - T2g;
+		    }
+	       }
+	  }
+	  {
+	       E Ti, T4Z, T1e, T53, Tl, T52, T1h, T50, Tp, T56, T1n, T5a, Ts, T59, T1q;
+	       E T57;
+	       {
+		    E Tg, Th, T1c, T1d;
+		    Tg = rio[WS(ios, 2)];
+		    Th = iio[-WS(ios, 18)];
+		    Ti = Tg + Th;
+		    T4Z = Tg - Th;
+		    T1c = iio[-WS(ios, 2)];
+		    T1d = rio[WS(ios, 18)];
+		    T1e = T1c - T1d;
+		    T53 = T1c + T1d;
+	       }
+	       {
+		    E Tj, Tk, T1f, T1g;
+		    Tj = rio[WS(ios, 10)];
+		    Tk = iio[-WS(ios, 26)];
+		    Tl = Tj + Tk;
+		    T52 = Tj - Tk;
+		    T1f = iio[-WS(ios, 10)];
+		    T1g = rio[WS(ios, 26)];
+		    T1h = T1f - T1g;
+		    T50 = T1f + T1g;
+	       }
+	       {
+		    E Tn, To, T1l, T1m;
+		    Tn = iio[-WS(ios, 30)];
+		    To = rio[WS(ios, 14)];
+		    Tp = Tn + To;
+		    T56 = Tn - To;
+		    T1l = iio[-WS(ios, 14)];
+		    T1m = rio[WS(ios, 30)];
+		    T1n = T1l - T1m;
+		    T5a = T1m + T1l;
+	       }
+	       {
+		    E Tq, Tr, T1o, T1p;
+		    Tq = rio[WS(ios, 6)];
+		    Tr = iio[-WS(ios, 22)];
+		    Ts = Tq + Tr;
+		    T59 = Tq - Tr;
+		    T1o = iio[-WS(ios, 6)];
+		    T1p = rio[WS(ios, 22)];
+		    T1q = T1o - T1p;
+		    T57 = T1o + T1p;
+	       }
+	       {
+		    E Tm, Tt, T6B, T6C;
+		    Tm = Ti + Tl;
+		    Tt = Tp + Ts;
+		    Tu = Tm + Tt;
+		    T3D = Tt - Tm;
+		    T6B = T53 - T52;
+		    T6C = T4Z + T50;
+		    T6D = FNMS(KP382683432, T6C, KP923879532 * T6B);
+		    T73 = FMA(KP382683432, T6B, KP923879532 * T6C);
+	       }
+	       {
+		    E T6E, T6F, T1b, T1i;
+		    T6E = T56 + T57;
+		    T6F = T59 + T5a;
+		    T6G = FNMS(KP923879532, T6F, KP382683432 * T6E);
+		    T74 = FMA(KP923879532, T6E, KP382683432 * T6F);
+		    T1b = Ti - Tl;
+		    T1i = T1e - T1h;
+		    T1j = T1b + T1i;
+		    T2d = T1i - T1b;
+	       }
+	       {
+		    E T1k, T1r, T51, T54;
+		    T1k = Tp - Ts;
+		    T1r = T1n - T1q;
+		    T1s = T1k - T1r;
+		    T2e = T1k + T1r;
+		    T51 = T4Z - T50;
+		    T54 = T52 + T53;
+		    T55 = FNMS(KP382683432, T54, KP923879532 * T51);
+		    T5z = FMA(KP923879532, T54, KP382683432 * T51);
+	       }
+	       {
+		    E T58, T5b, T3j, T3k;
+		    T58 = T56 - T57;
+		    T5b = T59 - T5a;
+		    T5c = FMA(KP923879532, T58, KP382683432 * T5b);
+		    T5A = FNMS(KP382683432, T58, KP923879532 * T5b);
+		    T3j = T1e + T1h;
+		    T3k = T1q + T1n;
+		    T3l = T3j - T3k;
+		    T48 = T3j + T3k;
+	       }
+	  }
+	  {
+	       E Ty, T4t, T1H, T4y, TB, T4x, T1K, T4u, TI, T4B, T1B, T4o, TF, T4A, T1y;
+	       E T4r;
+	       {
+		    E Tw, Tx, T1I, T1J;
+		    Tw = rio[WS(ios, 1)];
+		    Tx = iio[-WS(ios, 17)];
+		    Ty = Tw + Tx;
+		    T4t = Tw - Tx;
+		    {
+			 E T1F, T1G, Tz, TA;
+			 T1F = iio[-WS(ios, 1)];
+			 T1G = rio[WS(ios, 17)];
+			 T1H = T1F - T1G;
+			 T4y = T1F + T1G;
+			 Tz = rio[WS(ios, 9)];
+			 TA = iio[-WS(ios, 25)];
+			 TB = Tz + TA;
+			 T4x = Tz - TA;
+		    }
+		    T1I = iio[-WS(ios, 9)];
+		    T1J = rio[WS(ios, 25)];
+		    T1K = T1I - T1J;
+		    T4u = T1I + T1J;
+		    {
+			 E TG, TH, T4m, T1z, T1A, T4n;
+			 TG = iio[-WS(ios, 29)];
+			 TH = rio[WS(ios, 13)];
+			 T4m = TG - TH;
+			 T1z = iio[-WS(ios, 13)];
+			 T1A = rio[WS(ios, 29)];
+			 T4n = T1A + T1z;
+			 TI = TG + TH;
+			 T4B = T4m + T4n;
+			 T1B = T1z - T1A;
+			 T4o = T4m - T4n;
+		    }
+		    {
+			 E TD, TE, T4q, T1w, T1x, T4p;
+			 TD = rio[WS(ios, 5)];
+			 TE = iio[-WS(ios, 21)];
+			 T4q = TD - TE;
+			 T1w = iio[-WS(ios, 5)];
+			 T1x = rio[WS(ios, 21)];
+			 T4p = T1w + T1x;
+			 TF = TD + TE;
+			 T4A = T4q + T4p;
+			 T1y = T1w - T1x;
+			 T4r = T4p - T4q;
+		    }
+	       }
+	       {
+		    E TC, TJ, T6J, T6K;
+		    TC = Ty + TB;
+		    TJ = TF + TI;
+		    TK = TC + TJ;
+		    T3n = TC - TJ;
+		    T6J = T4y - T4x;
+		    T6K = KP707106781 * (T4r + T4o);
+		    T6L = T6J + T6K;
+		    T7t = T6J - T6K;
+	       }
+	       {
+		    E T6M, T6N, T1v, T1C;
+		    T6M = KP707106781 * (T4A + T4B);
+		    T6N = T4t + T4u;
+		    T6O = T6M + T6N;
+		    T7s = T6N - T6M;
+		    T1v = Ty - TB;
+		    T1C = T1y - T1B;
+		    T1D = T1v + T1C;
+		    T2L = T1v - T1C;
+	       }
+	       {
+		    E T1E, T1L, T4s, T4v;
+		    T1E = TI - TF;
+		    T1L = T1H - T1K;
+		    T1M = T1E + T1L;
+		    T2M = T1L - T1E;
+		    T4s = KP707106781 * (T4o - T4r);
+		    T4v = T4t - T4u;
+		    T4w = T4s + T4v;
+		    T62 = T4v - T4s;
+	       }
+	       {
+		    E T4z, T4C, T3o, T3p;
+		    T4z = T4x + T4y;
+		    T4C = KP707106781 * (T4A - T4B);
+		    T4D = T4z + T4C;
+		    T61 = T4z - T4C;
+		    T3o = T1H + T1K;
+		    T3p = T1y + T1B;
+		    T3q = T3o - T3p;
+		    T41 = T3p + T3o;
+	       }
+	  }
+	  {
+	       E TN, T4T, T20, T4N, TQ, T4M, T23, T4U, TX, T4Q, T1U, T4K, TU, T4R, T1R;
+	       E T4H;
+	       {
+		    E TL, TM, T21, T22;
+		    TL = iio[-WS(ios, 31)];
+		    TM = rio[WS(ios, 15)];
+		    TN = TL + TM;
+		    T4T = TL - TM;
+		    {
+			 E T1Y, T1Z, TO, TP;
+			 T1Y = iio[-WS(ios, 15)];
+			 T1Z = rio[WS(ios, 31)];
+			 T20 = T1Y - T1Z;
+			 T4N = T1Z + T1Y;
+			 TO = rio[WS(ios, 7)];
+			 TP = iio[-WS(ios, 23)];
+			 TQ = TO + TP;
+			 T4M = TO - TP;
+		    }
+		    T21 = iio[-WS(ios, 7)];
+		    T22 = rio[WS(ios, 23)];
+		    T23 = T21 - T22;
+		    T4U = T21 + T22;
+		    {
+			 E TV, TW, T4I, T1S, T1T, T4J;
+			 TV = iio[-WS(ios, 27)];
+			 TW = rio[WS(ios, 11)];
+			 T4I = TV - TW;
+			 T1S = iio[-WS(ios, 11)];
+			 T1T = rio[WS(ios, 27)];
+			 T4J = T1T + T1S;
+			 TX = TV + TW;
+			 T4Q = T4I - T4J;
+			 T1U = T1S - T1T;
+			 T4K = T4I + T4J;
+		    }
+		    {
+			 E TS, TT, T4F, T1P, T1Q, T4G;
+			 TS = rio[WS(ios, 3)];
+			 TT = iio[-WS(ios, 19)];
+			 T4F = TS - TT;
+			 T1P = iio[-WS(ios, 3)];
+			 T1Q = rio[WS(ios, 19)];
+			 T4G = T1P + T1Q;
+			 TU = TS + TT;
+			 T4R = T4G - T4F;
+			 T1R = T1P - T1Q;
+			 T4H = T4F + T4G;
+		    }
+	       }
+	       {
+		    E TR, TY, T6Q, T6R;
+		    TR = TN + TQ;
+		    TY = TU + TX;
+		    TZ = TR + TY;
+		    T3s = TR - TY;
+		    T6Q = KP707106781 * (T4R + T4Q);
+		    T6R = T4M + T4N;
+		    T6S = T6Q - T6R;
+		    T7w = T6Q + T6R;
+	       }
+	       {
+		    E T6T, T6U, T1O, T1V;
+		    T6T = KP707106781 * (T4H + T4K);
+		    T6U = T4T + T4U;
+		    T6V = T6T + T6U;
+		    T7v = T6U - T6T;
+		    T1O = TN - TQ;
+		    T1V = T1R - T1U;
+		    T1W = T1O + T1V;
+		    T2O = T1O - T1V;
+	       }
+	       {
+		    E T1X, T24, T4L, T4O;
+		    T1X = TX - TU;
+		    T24 = T20 - T23;
+		    T25 = T1X + T24;
+		    T2P = T24 - T1X;
+		    T4L = KP707106781 * (T4H - T4K);
+		    T4O = T4M - T4N;
+		    T4P = T4L + T4O;
+		    T64 = T4O - T4L;
+	       }
+	       {
+		    E T4S, T4V, T3t, T3u;
+		    T4S = KP707106781 * (T4Q - T4R);
+		    T4V = T4T - T4U;
+		    T4W = T4S + T4V;
+		    T65 = T4V - T4S;
+		    T3t = T20 + T23;
+		    T3u = T1R + T1U;
+		    T3v = T3t - T3u;
+		    T42 = T3u + T3t;
+	       }
+	  }
+	  {
+	       E Tv, T10, T4g, T4i, T4j, T4k, T4f, T4h;
+	       Tv = Tf + Tu;
+	       T10 = TK + TZ;
+	       T4g = Tv - T10;
+	       T4i = T48 + T47;
+	       T4j = T41 + T42;
+	       T4k = T4i - T4j;
+	       rio[0] = Tv + T10;
+	       iio[-WS(ios, 31)] = T4j + T4i;
+	       T4f = W[30];
+	       T4h = W[31];
+	       rio[WS(ios, 16)] = FNMS(T4h, T4k, T4f * T4g);
+	       iio[-WS(ios, 15)] = FMA(T4h, T4g, T4f * T4k);
+	  }
+	  {
+	       E T44, T4c, T4a, T4e;
+	       {
+		    E T40, T43, T46, T49;
+		    T40 = Tf - Tu;
+		    T43 = T41 - T42;
+		    T44 = T40 + T43;
+		    T4c = T40 - T43;
+		    T46 = TZ - TK;
+		    T49 = T47 - T48;
+		    T4a = T46 + T49;
+		    T4e = T49 - T46;
+	       }
+	       {
+		    E T3Z, T45, T4b, T4d;
+		    T3Z = W[46];
+		    T45 = W[47];
+		    rio[WS(ios, 24)] = FNMS(T45, T4a, T3Z * T44);
+		    iio[-WS(ios, 7)] = FMA(T45, T44, T3Z * T4a);
+		    T4b = W[14];
+		    T4d = W[15];
+		    rio[WS(ios, 8)] = FNMS(T4d, T4e, T4b * T4c);
+		    iio[-WS(ios, 23)] = FMA(T4d, T4c, T4b * T4e);
+	       }
+	  }
+	  {
+	       E T3m, T3H, T3T, T3O, T3C, T3P, T3x, T3S;
+	       T3m = T3i + T3l;
+	       T3H = T3D + T3G;
+	       T3T = T3G - T3D;
+	       T3O = T3i - T3l;
+	       {
+		    E T3A, T3B, T3r, T3w;
+		    T3A = T3q - T3n;
+		    T3B = T3s + T3v;
+		    T3C = KP707106781 * (T3A + T3B);
+		    T3P = KP707106781 * (T3B - T3A);
+		    T3r = T3n + T3q;
+		    T3w = T3s - T3v;
+		    T3x = KP707106781 * (T3r + T3w);
+		    T3S = KP707106781 * (T3r - T3w);
+	       }
+	       {
+		    E T3y, T3I, T3h, T3z;
+		    T3y = T3m + T3x;
+		    T3I = T3C + T3H;
+		    T3h = W[54];
+		    T3z = W[55];
+		    rio[WS(ios, 28)] = FNMS(T3z, T3I, T3h * T3y);
+		    iio[-WS(ios, 3)] = FMA(T3z, T3y, T3h * T3I);
+	       }
+	       {
+		    E T3W, T3Y, T3V, T3X;
+		    T3W = T3O - T3P;
+		    T3Y = T3T - T3S;
+		    T3V = W[38];
+		    T3X = W[39];
+		    rio[WS(ios, 20)] = FNMS(T3X, T3Y, T3V * T3W);
+		    iio[-WS(ios, 11)] = FMA(T3X, T3W, T3V * T3Y);
+	       }
+	       {
+		    E T3K, T3M, T3J, T3L;
+		    T3K = T3m - T3x;
+		    T3M = T3H - T3C;
+		    T3J = W[22];
+		    T3L = W[23];
+		    rio[WS(ios, 12)] = FNMS(T3L, T3M, T3J * T3K);
+		    iio[-WS(ios, 19)] = FMA(T3L, T3K, T3J * T3M);
+	       }
+	       {
+		    E T3Q, T3U, T3N, T3R;
+		    T3Q = T3O + T3P;
+		    T3U = T3S + T3T;
+		    T3N = W[6];
+		    T3R = W[7];
+		    rio[WS(ios, 4)] = FNMS(T3R, T3U, T3N * T3Q);
+		    iio[-WS(ios, 27)] = FMA(T3R, T3Q, T3N * T3U);
+	       }
+	  }
+	  {
+	       E T2K, T36, T2Z, T3b, T2R, T3a, T2W, T37, T2J, T2X;
+	       T2J = KP707106781 * (T2e - T2d);
+	       T2K = T2I + T2J;
+	       T36 = T2I - T2J;
+	       T2X = KP707106781 * (T1j - T1s);
+	       T2Z = T2X + T2Y;
+	       T3b = T2Y - T2X;
+	       {
+		    E T2N, T2Q, T2U, T2V;
+		    T2N = FNMS(KP382683432, T2M, KP923879532 * T2L);
+		    T2Q = FMA(KP923879532, T2O, KP382683432 * T2P);
+		    T2R = T2N + T2Q;
+		    T3a = T2Q - T2N;
+		    T2U = FMA(KP382683432, T2L, KP923879532 * T2M);
+		    T2V = FNMS(KP382683432, T2O, KP923879532 * T2P);
+		    T2W = T2U + T2V;
+		    T37 = T2U - T2V;
+	       }
+	       {
+		    E T2S, T30, T2H, T2T;
+		    T2S = T2K + T2R;
+		    T30 = T2W + T2Z;
+		    T2H = W[2];
+		    T2T = W[3];
+		    rio[WS(ios, 2)] = FNMS(T2T, T30, T2H * T2S);
+		    iio[-WS(ios, 29)] = FMA(T2T, T2S, T2H * T30);
+	       }
+	       {
+		    E T3e, T3g, T3d, T3f;
+		    T3e = T36 - T37;
+		    T3g = T3b - T3a;
+		    T3d = W[18];
+		    T3f = W[19];
+		    rio[WS(ios, 10)] = FNMS(T3f, T3g, T3d * T3e);
+		    iio[-WS(ios, 21)] = FMA(T3f, T3e, T3d * T3g);
+	       }
+	       {
+		    E T32, T34, T31, T33;
+		    T32 = T2K - T2R;
+		    T34 = T2Z - T2W;
+		    T31 = W[34];
+		    T33 = W[35];
+		    rio[WS(ios, 18)] = FNMS(T33, T34, T31 * T32);
+		    iio[-WS(ios, 13)] = FMA(T33, T32, T31 * T34);
+	       }
+	       {
+		    E T38, T3c, T35, T39;
+		    T38 = T36 + T37;
+		    T3c = T3a + T3b;
+		    T35 = W[50];
+		    T39 = W[51];
+		    rio[WS(ios, 26)] = FNMS(T39, T3c, T35 * T38);
+		    iio[-WS(ios, 5)] = FMA(T39, T38, T35 * T3c);
+	       }
+	  }
+	  {
+	       E T1u, T2w, T2p, T2B, T27, T2A, T2c, T2x, T1t, T2f;
+	       T1t = KP707106781 * (T1j + T1s);
+	       T1u = T1a + T1t;
+	       T2w = T1a - T1t;
+	       T2f = KP707106781 * (T2d + T2e);
+	       T2p = T2f + T2o;
+	       T2B = T2o - T2f;
+	       {
+		    E T1N, T26, T2a, T2b;
+		    T1N = FMA(KP923879532, T1D, KP382683432 * T1M);
+		    T26 = FNMS(KP382683432, T25, KP923879532 * T1W);
+		    T27 = T1N + T26;
+		    T2A = T1N - T26;
+		    T2a = FNMS(KP382683432, T1D, KP923879532 * T1M);
+		    T2b = FMA(KP382683432, T1W, KP923879532 * T25);
+		    T2c = T2a + T2b;
+		    T2x = T2b - T2a;
+	       }
+	       {
+		    E T28, T2q, T11, T29;
+		    T28 = T1u + T27;
+		    T2q = T2c + T2p;
+		    T11 = W[58];
+		    T29 = W[59];
+		    rio[WS(ios, 30)] = FNMS(T29, T2q, T11 * T28);
+		    iio[-WS(ios, 1)] = FMA(T29, T28, T11 * T2q);
+	       }
+	       {
+		    E T2E, T2G, T2D, T2F;
+		    T2E = T2w - T2x;
+		    T2G = T2B - T2A;
+		    T2D = W[42];
+		    T2F = W[43];
+		    rio[WS(ios, 22)] = FNMS(T2F, T2G, T2D * T2E);
+		    iio[-WS(ios, 9)] = FMA(T2F, T2E, T2D * T2G);
+	       }
+	       {
+		    E T2s, T2u, T2r, T2t;
+		    T2s = T1u - T27;
+		    T2u = T2p - T2c;
+		    T2r = W[26];
+		    T2t = W[27];
+		    rio[WS(ios, 14)] = FNMS(T2t, T2u, T2r * T2s);
+		    iio[-WS(ios, 17)] = FMA(T2t, T2s, T2r * T2u);
+	       }
+	       {
+		    E T2y, T2C, T2v, T2z;
+		    T2y = T2w + T2x;
+		    T2C = T2A + T2B;
+		    T2v = W[10];
+		    T2z = W[11];
+		    rio[WS(ios, 6)] = FNMS(T2z, T2C, T2v * T2y);
+		    iio[-WS(ios, 25)] = FMA(T2z, T2y, T2v * T2C);
+	       }
+	  }
+	  {
+	       E T4Y, T5N, T5F, T5Q, T5p, T5R, T5C, T5M;
+	       {
+		    E T4E, T4X, T5D, T5E;
+		    T4E = FNMS(KP195090322, T4D, KP980785280 * T4w);
+		    T4X = FMA(KP195090322, T4P, KP980785280 * T4W);
+		    T4Y = T4E + T4X;
+		    T5N = T4X - T4E;
+		    T5D = FMA(KP980785280, T4D, KP195090322 * T4w);
+		    T5E = FNMS(KP195090322, T4W, KP980785280 * T4P);
+		    T5F = T5D + T5E;
+		    T5Q = T5D - T5E;
+	       }
+	       {
+		    E T5d, T5o, T5y, T5B;
+		    T5d = T55 + T5c;
+		    T5o = T5k + T5n;
+		    T5p = T5d + T5o;
+		    T5R = T5o - T5d;
+		    T5y = T5u + T5x;
+		    T5B = T5z + T5A;
+		    T5C = T5y + T5B;
+		    T5M = T5y - T5B;
+	       }
+	       {
+		    E T5q, T5G, T4l, T5r;
+		    T5q = T4Y + T5p;
+		    T5G = T5C + T5F;
+		    T4l = W[0];
+		    T5r = W[1];
+		    rio[WS(ios, 1)] = FNMS(T5r, T5G, T4l * T5q);
+		    iio[-WS(ios, 30)] = FMA(T4l, T5G, T5r * T5q);
+	       }
+	       {
+		    E T5U, T5W, T5T, T5V;
+		    T5U = T5R - T5Q;
+		    T5W = T5M - T5N;
+		    T5T = W[16];
+		    T5V = W[17];
+		    rio[WS(ios, 9)] = FNMS(T5V, T5W, T5T * T5U);
+		    iio[-WS(ios, 22)] = FMA(T5T, T5W, T5V * T5U);
+	       }
+	       {
+		    E T5I, T5K, T5H, T5J;
+		    T5I = T5C - T5F;
+		    T5K = T5p - T4Y;
+		    T5H = W[32];
+		    T5J = W[33];
+		    iio[-WS(ios, 14)] = FMA(T5H, T5I, T5J * T5K);
+		    rio[WS(ios, 17)] = FNMS(T5J, T5I, T5H * T5K);
+	       }
+	       {
+		    E T5O, T5S, T5L, T5P;
+		    T5O = T5M + T5N;
+		    T5S = T5Q + T5R;
+		    T5L = W[48];
+		    T5P = W[49];
+		    iio[-WS(ios, 6)] = FMA(T5L, T5O, T5P * T5S);
+		    rio[WS(ios, 25)] = FNMS(T5P, T5O, T5L * T5S);
+	       }
+	  }
+	  {
+	       E T60, T6q, T6f, T6n, T67, T6m, T6c, T6r;
+	       {
+		    E T5Y, T5Z, T6d, T6e;
+		    T5Y = T5u - T5x;
+		    T5Z = T5c - T55;
+		    T60 = T5Y + T5Z;
+		    T6q = T5Y - T5Z;
+		    T6d = T5z - T5A;
+		    T6e = T5n - T5k;
+		    T6f = T6d + T6e;
+		    T6n = T6e - T6d;
+	       }
+	       {
+		    E T63, T66, T6a, T6b;
+		    T63 = FNMS(KP555570233, T62, KP831469612 * T61);
+		    T66 = FMA(KP831469612, T64, KP555570233 * T65);
+		    T67 = T63 + T66;
+		    T6m = T66 - T63;
+		    T6a = FMA(KP555570233, T61, KP831469612 * T62);
+		    T6b = FNMS(KP555570233, T64, KP831469612 * T65);
+		    T6c = T6a + T6b;
+		    T6r = T6a - T6b;
+	       }
+	       {
+		    E T68, T6g, T5X, T69;
+		    T68 = T60 + T67;
+		    T6g = T6c + T6f;
+		    T5X = W[56];
+		    T69 = W[57];
+		    iio[-WS(ios, 2)] = FMA(T5X, T68, T69 * T6g);
+		    rio[WS(ios, 29)] = FNMS(T69, T68, T5X * T6g);
+	       }
+	       {
+		    E T6u, T6w, T6t, T6v;
+		    T6u = T6q - T6r;
+		    T6w = T6n - T6m;
+		    T6t = W[40];
+		    T6v = W[41];
+		    iio[-WS(ios, 10)] = FMA(T6t, T6u, T6v * T6w);
+		    rio[WS(ios, 21)] = FNMS(T6v, T6u, T6t * T6w);
+	       }
+	       {
+		    E T6i, T6k, T6h, T6j;
+		    T6i = T6f - T6c;
+		    T6k = T60 - T67;
+		    T6h = W[24];
+		    T6j = W[25];
+		    rio[WS(ios, 13)] = FNMS(T6j, T6k, T6h * T6i);
+		    iio[-WS(ios, 18)] = FMA(T6h, T6k, T6j * T6i);
+	       }
+	       {
+		    E T6o, T6s, T6l, T6p;
+		    T6o = T6m + T6n;
+		    T6s = T6q + T6r;
+		    T6l = W[8];
+		    T6p = W[9];
+		    rio[WS(ios, 5)] = FNMS(T6p, T6s, T6l * T6o);
+		    iio[-WS(ios, 26)] = FMA(T6l, T6s, T6p * T6o);
+	       }
+	  }
+	  {
+	       E T7y, T7R, T7J, T7U, T7B, T7V, T7G, T7Q;
+	       {
+		    E T7u, T7x, T7H, T7I;
+		    T7u = FNMS(KP555570233, T7t, KP831469612 * T7s);
+		    T7x = FNMS(KP555570233, T7w, KP831469612 * T7v);
+		    T7y = T7u + T7x;
+		    T7R = T7x - T7u;
+		    T7H = FMA(KP831469612, T7t, KP555570233 * T7s);
+		    T7I = FMA(KP831469612, T7w, KP555570233 * T7v);
+		    T7J = T7H - T7I;
+		    T7U = T7H + T7I;
+	       }
+	       {
+		    E T7z, T7A, T7E, T7F;
+		    T7z = T6G - T6D;
+		    T7A = T77 - T76;
+		    T7B = T7z + T7A;
+		    T7V = T7A - T7z;
+		    T7E = T6y - T6z;
+		    T7F = T73 - T74;
+		    T7G = T7E + T7F;
+		    T7Q = T7E - T7F;
+	       }
+	       {
+		    E T7C, T7K, T7r, T7D;
+		    T7C = T7y + T7B;
+		    T7K = T7G + T7J;
+		    T7r = W[4];
+		    T7D = W[5];
+		    rio[WS(ios, 3)] = FNMS(T7D, T7K, T7r * T7C);
+		    iio[-WS(ios, 28)] = FMA(T7r, T7K, T7D * T7C);
+	       }
+	       {
+		    E T7Y, T80, T7X, T7Z;
+		    T7Y = T7V - T7U;
+		    T80 = T7Q - T7R;
+		    T7X = W[20];
+		    T7Z = W[21];
+		    rio[WS(ios, 11)] = FNMS(T7Z, T80, T7X * T7Y);
+		    iio[-WS(ios, 20)] = FMA(T7X, T80, T7Z * T7Y);
+	       }
+	       {
+		    E T7M, T7O, T7L, T7N;
+		    T7M = T7G - T7J;
+		    T7O = T7B - T7y;
+		    T7L = W[36];
+		    T7N = W[37];
+		    iio[-WS(ios, 12)] = FMA(T7L, T7M, T7N * T7O);
+		    rio[WS(ios, 19)] = FNMS(T7N, T7M, T7L * T7O);
+	       }
+	       {
+		    E T7S, T7W, T7P, T7T;
+		    T7S = T7Q + T7R;
+		    T7W = T7U + T7V;
+		    T7P = W[52];
+		    T7T = W[53];
+		    iio[-WS(ios, 4)] = FMA(T7P, T7S, T7T * T7W);
+		    rio[WS(ios, 27)] = FNMS(T7T, T7S, T7P * T7W);
+	       }
+	  }
+	  {
+	       E T6I, T7k, T79, T7h, T6X, T7g, T72, T7l;
+	       {
+		    E T6A, T6H, T75, T78;
+		    T6A = T6y + T6z;
+		    T6H = T6D + T6G;
+		    T6I = T6A + T6H;
+		    T7k = T6A - T6H;
+		    T75 = T73 + T74;
+		    T78 = T76 + T77;
+		    T79 = T75 + T78;
+		    T7h = T78 - T75;
+	       }
+	       {
+		    E T6P, T6W, T70, T71;
+		    T6P = FNMS(KP195090322, T6O, KP980785280 * T6L);
+		    T6W = FMA(KP980785280, T6S, KP195090322 * T6V);
+		    T6X = T6P + T6W;
+		    T7g = T6W - T6P;
+		    T70 = FMA(KP195090322, T6L, KP980785280 * T6O);
+		    T71 = FNMS(KP195090322, T6S, KP980785280 * T6V);
+		    T72 = T70 + T71;
+		    T7l = T70 - T71;
+	       }
+	       {
+		    E T6Y, T7a, T6x, T6Z;
+		    T6Y = T6I + T6X;
+		    T7a = T72 + T79;
+		    T6x = W[60];
+		    T6Z = W[61];
+		    iio[0] = FMA(T6x, T6Y, T6Z * T7a);
+		    rio[WS(ios, 31)] = FNMS(T6Z, T6Y, T6x * T7a);
+	       }
+	       {
+		    E T7o, T7q, T7n, T7p;
+		    T7o = T7k - T7l;
+		    T7q = T7h - T7g;
+		    T7n = W[44];
+		    T7p = W[45];
+		    iio[-WS(ios, 8)] = FMA(T7n, T7o, T7p * T7q);
+		    rio[WS(ios, 23)] = FNMS(T7p, T7o, T7n * T7q);
+	       }
+	       {
+		    E T7c, T7e, T7b, T7d;
+		    T7c = T79 - T72;
+		    T7e = T6I - T6X;
+		    T7b = W[28];
+		    T7d = W[29];
+		    rio[WS(ios, 15)] = FNMS(T7d, T7e, T7b * T7c);
+		    iio[-WS(ios, 16)] = FMA(T7b, T7e, T7d * T7c);
+	       }
+	       {
+		    E T7i, T7m, T7f, T7j;
+		    T7i = T7g + T7h;
+		    T7m = T7k + T7l;
+		    T7f = W[12];
+		    T7j = W[13];
+		    rio[WS(ios, 7)] = FNMS(T7j, T7m, T7f * T7i);
+		    iio[-WS(ios, 24)] = FMA(T7f, T7m, T7j * T7i);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 32},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 32, "hb_32", twinstr, {340, 114, 94, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_32) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_32, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_4.c b/src/fftw3/rdft/codelets/hc2r/hb_4.c
new file mode 100644
index 0000000..ac05b69
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_4.c
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:22 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 4 -dif -name hb_4 -include hb.h */
+
+/*
+ * This function contains 22 FP additions, 12 FP multiplications,
+ * (or, 16 additions, 6 multiplications, 6 fused multiply/add),
+ * 13 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_4.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_4.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_4.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_4(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 6) {
+	  E T3, Ti, Tc, Tn, T6, Tm, Tf, Tj;
+	  {
+	       E T1, T2, Ta, Tb;
+	       T1 = rio[0];
+	       T2 = iio[-WS(ios, 2)];
+	       T3 = T1 + T2;
+	       Ti = T1 - T2;
+	       Ta = iio[0];
+	       Tb = rio[WS(ios, 2)];
+	       Tc = Ta - Tb;
+	       Tn = Ta + Tb;
+	  }
+	  {
+	       E T4, T5, Td, Te;
+	       T4 = rio[WS(ios, 1)];
+	       T5 = iio[-WS(ios, 3)];
+	       T6 = T4 + T5;
+	       Tm = T4 - T5;
+	       Td = iio[-WS(ios, 1)];
+	       Te = rio[WS(ios, 3)];
+	       Tf = Td - Te;
+	       Tj = Td + Te;
+	  }
+	  rio[0] = T3 + T6;
+	  iio[-WS(ios, 3)] = Tc + Tf;
+	  {
+	       E Tq, Ts, Tp, Tr;
+	       Tq = Tn - Tm;
+	       Ts = Ti + Tj;
+	       Tp = W[4];
+	       Tr = W[5];
+	       iio[0] = FMA(Tp, Tq, Tr * Ts);
+	       rio[WS(ios, 3)] = FNMS(Tr, Tq, Tp * Ts);
+	  }
+	  {
+	       E T8, Tg, T7, T9;
+	       T8 = T3 - T6;
+	       Tg = Tc - Tf;
+	       T7 = W[2];
+	       T9 = W[3];
+	       rio[WS(ios, 2)] = FNMS(T9, Tg, T7 * T8);
+	       iio[-WS(ios, 1)] = FMA(T9, T8, T7 * Tg);
+	  }
+	  {
+	       E Tk, To, Th, Tl;
+	       Tk = Ti - Tj;
+	       To = Tm + Tn;
+	       Th = W[0];
+	       Tl = W[1];
+	       rio[WS(ios, 1)] = FNMS(Tl, To, Th * Tk);
+	       iio[-WS(ios, 2)] = FMA(Th, To, Tl * Tk);
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 4},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 4, "hb_4", twinstr, {16, 6, 6, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_4) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_4, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_5.c b/src/fftw3/rdft/codelets/hc2r/hb_5.c
new file mode 100644
index 0000000..e0af24d
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_5.c
@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:23 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 5 -dif -name hb_5 -include hb.h */
+
+/*
+ * This function contains 40 FP additions, 28 FP multiplications,
+ * (or, 26 additions, 14 multiplications, 14 fused multiply/add),
+ * 27 stack variables, and 20 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_5.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_5.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_5.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_5(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 8) {
+	  E T1, Tj, TG, Ts, T8, Ti, T9, Tn, TD, Tu, Tg, Tt;
+	  {
+	       E T7, Tr, T4, Tq;
+	       T1 = rio[0];
+	       {
+		    E T5, T6, T2, T3;
+		    T5 = rio[WS(ios, 2)];
+		    T6 = iio[-WS(ios, 3)];
+		    T7 = T5 + T6;
+		    Tr = T5 - T6;
+		    T2 = rio[WS(ios, 1)];
+		    T3 = iio[-WS(ios, 4)];
+		    T4 = T2 + T3;
+		    Tq = T2 - T3;
+	       }
+	       Tj = KP559016994 * (T4 - T7);
+	       TG = FMA(KP951056516, Tq, KP587785252 * Tr);
+	       Ts = FNMS(KP951056516, Tr, KP587785252 * Tq);
+	       T8 = T4 + T7;
+	       Ti = FNMS(KP250000000, T8, T1);
+	  }
+	  {
+	       E Tf, Tm, Tc, Tl;
+	       T9 = iio[0];
+	       {
+		    E Td, Te, Ta, Tb;
+		    Td = iio[-WS(ios, 2)];
+		    Te = rio[WS(ios, 3)];
+		    Tf = Td - Te;
+		    Tm = Td + Te;
+		    Ta = iio[-WS(ios, 1)];
+		    Tb = rio[WS(ios, 4)];
+		    Tc = Ta - Tb;
+		    Tl = Ta + Tb;
+	       }
+	       Tn = FNMS(KP951056516, Tm, KP587785252 * Tl);
+	       TD = FMA(KP951056516, Tl, KP587785252 * Tm);
+	       Tu = KP559016994 * (Tc - Tf);
+	       Tg = Tc + Tf;
+	       Tt = FNMS(KP250000000, Tg, T9);
+	  }
+	  rio[0] = T1 + T8;
+	  iio[-WS(ios, 4)] = T9 + Tg;
+	  {
+	       E TE, TM, TI, TK, TC, TH;
+	       TC = Tj + Ti;
+	       TE = TC - TD;
+	       TM = TC + TD;
+	       TH = Tu + Tt;
+	       TI = TG + TH;
+	       TK = TH - TG;
+	       {
+		    E TB, TF, TJ, TL;
+		    TB = W[0];
+		    TF = W[1];
+		    rio[WS(ios, 1)] = FNMS(TF, TI, TB * TE);
+		    iio[-WS(ios, 3)] = FMA(TB, TI, TF * TE);
+		    TJ = W[6];
+		    TL = W[7];
+		    iio[0] = FMA(TJ, TK, TL * TM);
+		    rio[WS(ios, 4)] = FNMS(TL, TK, TJ * TM);
+	       }
+	  }
+	  {
+	       E To, TA, Tw, Ty, Tk, Tv;
+	       Tk = Ti - Tj;
+	       To = Tk - Tn;
+	       TA = Tk + Tn;
+	       Tv = Tt - Tu;
+	       Tw = Ts + Tv;
+	       Ty = Tv - Ts;
+	       {
+		    E Th, Tp, Tx, Tz;
+		    Th = W[2];
+		    Tp = W[3];
+		    rio[WS(ios, 2)] = FNMS(Tp, Tw, Th * To);
+		    iio[-WS(ios, 2)] = FMA(Th, Tw, Tp * To);
+		    Tx = W[4];
+		    Tz = W[5];
+		    iio[-WS(ios, 1)] = FMA(Tx, Ty, Tz * TA);
+		    rio[WS(ios, 3)] = FNMS(Tz, Ty, Tx * TA);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 5},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 5, "hb_5", twinstr, {26, 14, 14, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_5) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_5, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_6.c b/src/fftw3/rdft/codelets/hc2r/hb_6.c
new file mode 100644
index 0000000..1a24696
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_6.c
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:25 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 6 -dif -name hb_6 -include hb.h */
+
+/*
+ * This function contains 46 FP additions, 28 FP multiplications,
+ * (or, 32 additions, 14 multiplications, 14 fused multiply/add),
+ * 25 stack variables, and 24 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_6.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_6.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_6.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_6(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 10) {
+	  E T3, Ty, Tp, TE, Ta, TO, Tm, TB, Tj, TL, Tq, TH;
+	  {
+	       E T1, T2, Tn, To;
+	       T1 = rio[0];
+	       T2 = iio[-WS(ios, 3)];
+	       T3 = T1 + T2;
+	       Ty = T1 - T2;
+	       Tn = iio[0];
+	       To = rio[WS(ios, 3)];
+	       Tp = Tn - To;
+	       TE = Tn + To;
+	  }
+	  {
+	       E T6, Tz, T9, TA;
+	       {
+		    E T4, T5, T7, T8;
+		    T4 = rio[WS(ios, 2)];
+		    T5 = iio[-WS(ios, 5)];
+		    T6 = T4 + T5;
+		    Tz = T4 - T5;
+		    T7 = iio[-WS(ios, 4)];
+		    T8 = rio[WS(ios, 1)];
+		    T9 = T7 + T8;
+		    TA = T7 - T8;
+	       }
+	       Ta = T6 + T9;
+	       TO = KP866025403 * (Tz - TA);
+	       Tm = KP866025403 * (T6 - T9);
+	       TB = Tz + TA;
+	  }
+	  {
+	       E Tf, TF, Ti, TG;
+	       {
+		    E Td, Te, Tg, Th;
+		    Td = iio[-WS(ios, 1)];
+		    Te = rio[WS(ios, 4)];
+		    Tf = Td - Te;
+		    TF = Te + Td;
+		    Tg = iio[-WS(ios, 2)];
+		    Th = rio[WS(ios, 5)];
+		    Ti = Tg - Th;
+		    TG = Tg + Th;
+	       }
+	       Tj = KP866025403 * (Tf - Ti);
+	       TL = KP866025403 * (TF + TG);
+	       Tq = Tf + Ti;
+	       TH = TF - TG;
+	  }
+	  rio[0] = T3 + Ta;
+	  iio[-WS(ios, 5)] = Tp + Tq;
+	  {
+	       E TC, TI, Tx, TD;
+	       TC = Ty + TB;
+	       TI = TE - TH;
+	       Tx = W[4];
+	       TD = W[5];
+	       rio[WS(ios, 3)] = FNMS(TD, TI, Tx * TC);
+	       iio[-WS(ios, 2)] = FMA(TD, TC, Tx * TI);
+	  }
+	  {
+	       E Tk, Tu, Ts, Tw, Tc, Tr;
+	       Tc = FNMS(KP500000000, Ta, T3);
+	       Tk = Tc + Tj;
+	       Tu = Tc - Tj;
+	       Tr = FNMS(KP500000000, Tq, Tp);
+	       Ts = Tm + Tr;
+	       Tw = Tr - Tm;
+	       {
+		    E Tb, Tl, Tt, Tv;
+		    Tb = W[6];
+		    Tl = W[7];
+		    rio[WS(ios, 4)] = FNMS(Tl, Ts, Tb * Tk);
+		    iio[-WS(ios, 1)] = FMA(Tl, Tk, Tb * Ts);
+		    Tt = W[2];
+		    Tv = W[3];
+		    rio[WS(ios, 2)] = FNMS(Tv, Tw, Tt * Tu);
+		    iio[-WS(ios, 3)] = FMA(Tv, Tu, Tt * Tw);
+	       }
+	  }
+	  {
+	       E TM, TU, TQ, TS, TK, TP;
+	       TK = FNMS(KP500000000, TB, Ty);
+	       TM = TK - TL;
+	       TU = TK + TL;
+	       TP = FMA(KP500000000, TH, TE);
+	       TQ = TO + TP;
+	       TS = TP - TO;
+	       {
+		    E TJ, TN, TR, TT;
+		    TJ = W[0];
+		    TN = W[1];
+		    rio[WS(ios, 1)] = FNMS(TN, TQ, TJ * TM);
+		    iio[-WS(ios, 4)] = FMA(TJ, TQ, TN * TM);
+		    TR = W[8];
+		    TT = W[9];
+		    iio[0] = FMA(TR, TS, TT * TU);
+		    rio[WS(ios, 5)] = FNMS(TT, TS, TR * TU);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 6},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 6, "hb_6", twinstr, {32, 14, 14, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_6) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_6, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_64.c b/src/fftw3/rdft/codelets/hc2r/hb_64.c
new file mode 100644
index 0000000..d3f9afc
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_64.c
@@ -0,0 +1,1972 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:42 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 64 -dif -name hb_64 -include hb.h */
+
+/*
+ * This function contains 1038 FP additions, 500 FP multiplications,
+ * (or, 808 additions, 270 multiplications, 230 fused multiply/add),
+ * 196 stack variables, and 256 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_64.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_64.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_64.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_64(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP634393284, +0.634393284163645498215171613225493370675687095);
+     DK(KP773010453, +0.773010453362736960810906609758469800971041293);
+     DK(KP098017140, +0.098017140329560601994195563888641845861136673);
+     DK(KP995184726, +0.995184726672196886244836953109479921575474869);
+     DK(KP471396736, +0.471396736825997648556387625905254377657460319);
+     DK(KP881921264, +0.881921264348355029712756863660388349508442621);
+     DK(KP290284677, +0.290284677254462367636192375817395274691476278);
+     DK(KP956940335, +0.956940335732208864935797886980269969482849206);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 126) {
+	  E Tf, T7i, Tfa, ThM, Tgp, ThH, T2c, T5O, T4T, T6n, Tcp, Ted, TcA, TdE, T87;
+	  E T9o, TK, T93, T2P, T4F, Tfo, Thz, T5T, T6j, Tbx, TdI, Tfl, ThA, T7r, T81;
+	  E TbE, TdH, TZ, T94, T38, T4G, Tfv, ThC, T5W, T6k, TbQ, TdK, Tfs, ThD, T7w;
+	  E T82, TbX, TdL, Tu, T84, Tfh, ThG, Tgm, ThN, T2v, T6m, T4K, T5P, Tce, TdF;
+	  E TcD, Tec, T7l, T9p, T1L, T20, T9c, T9d, T9e, T9f, T40, T66, Tg1, Thu, Tg8;
+	  E Thv, Tg5, Thr, T4n, T67, T4j, T69, T4w, T6a, TaT, TdW, Tb8, TdZ, TfU, Ths;
+	  E T7O, T8y, T7T, T8z, Tbc, TdX, Tbj, Te0, T1g, T1v, T97, T98, T99, T9a, T3j;
+	  E T5Z, TfI, Thk, TfP, Thl, TfM, Tho, T3G, T60, T3C, T62, T3P, T63, Tak, TdQ;
+	  E Tav, TdT, TfB, Thn, T7D, T8v, T7I, T8w, TaD, TdP, TaG, TdS;
+	  {
+	       E T3, Tcm, T4O, Tcv, T6, Tcu, T4R, Tcn, Td, Tcy, T2a, Tch, Ta, Tcx, T27;
+	       E Tck;
+	       {
+		    E T1, T2, T4P, T4Q;
+		    T1 = rio[0];
+		    T2 = iio[-WS(ios, 32)];
+		    T3 = T1 + T2;
+		    Tcm = T1 - T2;
+		    {
+			 E T4M, T4N, T4, T5;
+			 T4M = iio[0];
+			 T4N = rio[WS(ios, 32)];
+			 T4O = T4M - T4N;
+			 Tcv = T4M + T4N;
+			 T4 = rio[WS(ios, 16)];
+			 T5 = iio[-WS(ios, 48)];
+			 T6 = T4 + T5;
+			 Tcu = T4 - T5;
+		    }
+		    T4P = iio[-WS(ios, 16)];
+		    T4Q = rio[WS(ios, 48)];
+		    T4R = T4P - T4Q;
+		    Tcn = T4P + T4Q;
+		    {
+			 E Tb, Tc, Tcf, T28, T29, Tcg;
+			 Tb = iio[-WS(ios, 56)];
+			 Tc = rio[WS(ios, 24)];
+			 Tcf = Tb - Tc;
+			 T28 = iio[-WS(ios, 24)];
+			 T29 = rio[WS(ios, 56)];
+			 Tcg = T29 + T28;
+			 Td = Tb + Tc;
+			 Tcy = Tcf + Tcg;
+			 T2a = T28 - T29;
+			 Tch = Tcf - Tcg;
+		    }
+		    {
+			 E T8, T9, Tcj, T25, T26, Tci;
+			 T8 = rio[WS(ios, 8)];
+			 T9 = iio[-WS(ios, 40)];
+			 Tcj = T8 - T9;
+			 T25 = iio[-WS(ios, 8)];
+			 T26 = rio[WS(ios, 40)];
+			 Tci = T25 + T26;
+			 Ta = T8 + T9;
+			 Tcx = Tcj + Tci;
+			 T27 = T25 - T26;
+			 Tck = Tci - Tcj;
+		    }
+	       }
+	       {
+		    E T7, Te, Tf8, Tf9;
+		    T7 = T3 + T6;
+		    Te = Ta + Td;
+		    Tf = T7 + Te;
+		    T7i = T7 - Te;
+		    Tf8 = Tcv - Tcu;
+		    Tf9 = KP707106781 * (Tck + Tch);
+		    Tfa = Tf8 + Tf9;
+		    ThM = Tf8 - Tf9;
+	       }
+	       {
+		    E Tgn, Tgo, T24, T2b;
+		    Tgn = KP707106781 * (Tcx + Tcy);
+		    Tgo = Tcm + Tcn;
+		    Tgp = Tgn + Tgo;
+		    ThH = Tgo - Tgn;
+		    T24 = T3 - T6;
+		    T2b = T27 - T2a;
+		    T2c = T24 + T2b;
+		    T5O = T24 - T2b;
+	       }
+	       {
+		    E T4L, T4S, Tcl, Tco;
+		    T4L = Td - Ta;
+		    T4S = T4O - T4R;
+		    T4T = T4L + T4S;
+		    T6n = T4S - T4L;
+		    Tcl = KP707106781 * (Tch - Tck);
+		    Tco = Tcm - Tcn;
+		    Tcp = Tcl + Tco;
+		    Ted = Tco - Tcl;
+	       }
+	       {
+		    E Tcw, Tcz, T85, T86;
+		    Tcw = Tcu + Tcv;
+		    Tcz = KP707106781 * (Tcx - Tcy);
+		    TcA = Tcw + Tcz;
+		    TdE = Tcw - Tcz;
+		    T85 = T4O + T4R;
+		    T86 = T27 + T2a;
+		    T87 = T85 - T86;
+		    T9o = T86 + T85;
+	       }
+	  }
+	  {
+	       E TC, Tby, T2x, Tbu, T2N, Tbz, T7o, Tbv, TJ, TbB, TbC, T2E, T2G, Tbp, Tbs;
+	       E T7p, Tfj, Tfk;
+	       {
+		    E Tw, Tx, Ty, Tz, TA, TB;
+		    Tw = rio[WS(ios, 2)];
+		    Tx = iio[-WS(ios, 34)];
+		    Ty = Tw + Tx;
+		    Tz = rio[WS(ios, 18)];
+		    TA = iio[-WS(ios, 50)];
+		    TB = Tz + TA;
+		    TC = Ty + TB;
+		    Tby = Tz - TA;
+		    T2x = Ty - TB;
+		    Tbu = Tw - Tx;
+	       }
+	       {
+		    E T2H, T2I, T2J, T2K, T2L, T2M;
+		    T2H = iio[-WS(ios, 2)];
+		    T2I = rio[WS(ios, 34)];
+		    T2J = T2H - T2I;
+		    T2K = iio[-WS(ios, 18)];
+		    T2L = rio[WS(ios, 50)];
+		    T2M = T2K - T2L;
+		    T2N = T2J - T2M;
+		    Tbz = T2H + T2I;
+		    T7o = T2J + T2M;
+		    Tbv = T2K + T2L;
+	       }
+	       {
+		    E TF, Tbr, T2A, Tbq, TI, Tbn, T2D, Tbo;
+		    {
+			 E TD, TE, T2y, T2z;
+			 TD = rio[WS(ios, 10)];
+			 TE = iio[-WS(ios, 42)];
+			 TF = TD + TE;
+			 Tbr = TD - TE;
+			 T2y = iio[-WS(ios, 10)];
+			 T2z = rio[WS(ios, 42)];
+			 T2A = T2y - T2z;
+			 Tbq = T2y + T2z;
+		    }
+		    {
+			 E TG, TH, T2B, T2C;
+			 TG = iio[-WS(ios, 58)];
+			 TH = rio[WS(ios, 26)];
+			 TI = TG + TH;
+			 Tbn = TG - TH;
+			 T2B = iio[-WS(ios, 26)];
+			 T2C = rio[WS(ios, 58)];
+			 T2D = T2B - T2C;
+			 Tbo = T2C + T2B;
+		    }
+		    TJ = TF + TI;
+		    TbB = Tbr + Tbq;
+		    TbC = Tbn + Tbo;
+		    T2E = T2A - T2D;
+		    T2G = TI - TF;
+		    Tbp = Tbn - Tbo;
+		    Tbs = Tbq - Tbr;
+		    T7p = T2A + T2D;
+	       }
+	       TK = TC + TJ;
+	       T93 = T7p + T7o;
+	       {
+		    E T2F, T2O, Tfm, Tfn;
+		    T2F = T2x + T2E;
+		    T2O = T2G + T2N;
+		    T2P = FMA(KP923879532, T2F, KP382683432 * T2O);
+		    T4F = FNMS(KP382683432, T2F, KP923879532 * T2O);
+		    Tfm = KP707106781 * (TbB + TbC);
+		    Tfn = Tbu + Tbv;
+		    Tfo = Tfm + Tfn;
+		    Thz = Tfn - Tfm;
+	       }
+	       {
+		    E T5R, T5S, Tbt, Tbw;
+		    T5R = T2x - T2E;
+		    T5S = T2N - T2G;
+		    T5T = FNMS(KP382683432, T5S, KP923879532 * T5R);
+		    T6j = FMA(KP382683432, T5R, KP923879532 * T5S);
+		    Tbt = KP707106781 * (Tbp - Tbs);
+		    Tbw = Tbu - Tbv;
+		    Tbx = Tbt + Tbw;
+		    TdI = Tbw - Tbt;
+	       }
+	       Tfj = Tbz - Tby;
+	       Tfk = KP707106781 * (Tbs + Tbp);
+	       Tfl = Tfj + Tfk;
+	       ThA = Tfj - Tfk;
+	       {
+		    E T7n, T7q, TbA, TbD;
+		    T7n = TC - TJ;
+		    T7q = T7o - T7p;
+		    T7r = T7n + T7q;
+		    T81 = T7q - T7n;
+		    TbA = Tby + Tbz;
+		    TbD = KP707106781 * (TbB - TbC);
+		    TbE = TbA + TbD;
+		    TdH = TbA - TbD;
+	       }
+	  }
+	  {
+	       E TR, TbU, T2Q, TbN, T36, TbV, T7t, TbO, TY, TbR, TbS, T2X, T2Z, TbI, TbL;
+	       E T7u, Tfq, Tfr;
+	       {
+		    E TL, TM, TN, TO, TP, TQ;
+		    TL = iio[-WS(ios, 62)];
+		    TM = rio[WS(ios, 30)];
+		    TN = TL + TM;
+		    TO = rio[WS(ios, 14)];
+		    TP = iio[-WS(ios, 46)];
+		    TQ = TO + TP;
+		    TR = TN + TQ;
+		    TbU = TL - TM;
+		    T2Q = TN - TQ;
+		    TbN = TO - TP;
+	       }
+	       {
+		    E T30, T31, T32, T33, T34, T35;
+		    T30 = iio[-WS(ios, 30)];
+		    T31 = rio[WS(ios, 62)];
+		    T32 = T30 - T31;
+		    T33 = iio[-WS(ios, 14)];
+		    T34 = rio[WS(ios, 46)];
+		    T35 = T33 - T34;
+		    T36 = T32 - T35;
+		    TbV = T33 + T34;
+		    T7t = T32 + T35;
+		    TbO = T31 + T30;
+	       }
+	       {
+		    E TU, TbG, T2T, TbH, TX, TbJ, T2W, TbK;
+		    {
+			 E TS, TT, T2R, T2S;
+			 TS = rio[WS(ios, 6)];
+			 TT = iio[-WS(ios, 38)];
+			 TU = TS + TT;
+			 TbG = TS - TT;
+			 T2R = iio[-WS(ios, 6)];
+			 T2S = rio[WS(ios, 38)];
+			 T2T = T2R - T2S;
+			 TbH = T2R + T2S;
+		    }
+		    {
+			 E TV, TW, T2U, T2V;
+			 TV = iio[-WS(ios, 54)];
+			 TW = rio[WS(ios, 22)];
+			 TX = TV + TW;
+			 TbJ = TV - TW;
+			 T2U = iio[-WS(ios, 22)];
+			 T2V = rio[WS(ios, 54)];
+			 T2W = T2U - T2V;
+			 TbK = T2V + T2U;
+		    }
+		    TY = TU + TX;
+		    TbR = TbJ - TbK;
+		    TbS = TbH - TbG;
+		    T2X = T2T - T2W;
+		    T2Z = TX - TU;
+		    TbI = TbG + TbH;
+		    TbL = TbJ + TbK;
+		    T7u = T2T + T2W;
+	       }
+	       TZ = TR + TY;
+	       T94 = T7u + T7t;
+	       {
+		    E T2Y, T37, Tft, Tfu;
+		    T2Y = T2Q + T2X;
+		    T37 = T2Z + T36;
+		    T38 = FNMS(KP382683432, T37, KP923879532 * T2Y);
+		    T4G = FMA(KP382683432, T2Y, KP923879532 * T37);
+		    Tft = KP707106781 * (TbI + TbL);
+		    Tfu = TbU + TbV;
+		    Tfv = Tft + Tfu;
+		    ThC = Tfu - Tft;
+	       }
+	       {
+		    E T5U, T5V, TbM, TbP;
+		    T5U = T2Q - T2X;
+		    T5V = T36 - T2Z;
+		    T5W = FMA(KP923879532, T5U, KP382683432 * T5V);
+		    T6k = FNMS(KP382683432, T5U, KP923879532 * T5V);
+		    TbM = KP707106781 * (TbI - TbL);
+		    TbP = TbN - TbO;
+		    TbQ = TbM + TbP;
+		    TdK = TbP - TbM;
+	       }
+	       Tfq = KP707106781 * (TbS + TbR);
+	       Tfr = TbN + TbO;
+	       Tfs = Tfq - Tfr;
+	       ThD = Tfq + Tfr;
+	       {
+		    E T7s, T7v, TbT, TbW;
+		    T7s = TR - TY;
+		    T7v = T7t - T7u;
+		    T7w = T7s - T7v;
+		    T82 = T7s + T7v;
+		    TbT = KP707106781 * (TbR - TbS);
+		    TbW = TbU - TbV;
+		    TbX = TbT + TbW;
+		    TdL = TbW - TbT;
+	       }
+	  }
+	  {
+	       E Ti, T2g, Tl, T2j, T2d, T2k, Tfc, Tfb, Tc5, Tc2, Tp, T2p, Ts, T2s, T2m;
+	       E T2t, Tff, Tfe, Tcc, Tc9;
+	       {
+		    E Tc0, Tc4, Tc3, Tc1;
+		    {
+			 E Tg, Th, T2e, T2f;
+			 Tg = rio[WS(ios, 4)];
+			 Th = iio[-WS(ios, 36)];
+			 Ti = Tg + Th;
+			 Tc0 = Tg - Th;
+			 T2e = iio[-WS(ios, 4)];
+			 T2f = rio[WS(ios, 36)];
+			 T2g = T2e - T2f;
+			 Tc4 = T2e + T2f;
+		    }
+		    {
+			 E Tj, Tk, T2h, T2i;
+			 Tj = rio[WS(ios, 20)];
+			 Tk = iio[-WS(ios, 52)];
+			 Tl = Tj + Tk;
+			 Tc3 = Tj - Tk;
+			 T2h = iio[-WS(ios, 20)];
+			 T2i = rio[WS(ios, 52)];
+			 T2j = T2h - T2i;
+			 Tc1 = T2h + T2i;
+		    }
+		    T2d = Ti - Tl;
+		    T2k = T2g - T2j;
+		    Tfc = Tc0 + Tc1;
+		    Tfb = Tc4 - Tc3;
+		    Tc5 = Tc3 + Tc4;
+		    Tc2 = Tc0 - Tc1;
+	       }
+	       {
+		    E Tc7, Tcb, Tca, Tc8;
+		    {
+			 E Tn, To, T2n, T2o;
+			 Tn = iio[-WS(ios, 60)];
+			 To = rio[WS(ios, 28)];
+			 Tp = Tn + To;
+			 Tc7 = Tn - To;
+			 T2n = iio[-WS(ios, 28)];
+			 T2o = rio[WS(ios, 60)];
+			 T2p = T2n - T2o;
+			 Tcb = T2o + T2n;
+		    }
+		    {
+			 E Tq, Tr, T2q, T2r;
+			 Tq = rio[WS(ios, 12)];
+			 Tr = iio[-WS(ios, 44)];
+			 Ts = Tq + Tr;
+			 Tca = Tq - Tr;
+			 T2q = iio[-WS(ios, 12)];
+			 T2r = rio[WS(ios, 44)];
+			 T2s = T2q - T2r;
+			 Tc8 = T2q + T2r;
+		    }
+		    T2m = Tp - Ts;
+		    T2t = T2p - T2s;
+		    Tff = Tca + Tcb;
+		    Tfe = Tc7 + Tc8;
+		    Tcc = Tca - Tcb;
+		    Tc9 = Tc7 - Tc8;
+	       }
+	       {
+		    E Tm, Tt, Tfd, Tfg;
+		    Tm = Ti + Tl;
+		    Tt = Tp + Ts;
+		    Tu = Tm + Tt;
+		    T84 = Tt - Tm;
+		    Tfd = FNMS(KP382683432, Tfc, KP923879532 * Tfb);
+		    Tfg = FNMS(KP923879532, Tff, KP382683432 * Tfe);
+		    Tfh = Tfd + Tfg;
+		    ThG = Tfg - Tfd;
+	       }
+	       {
+		    E Tgk, Tgl, T2l, T2u;
+		    Tgk = FMA(KP382683432, Tfb, KP923879532 * Tfc);
+		    Tgl = FMA(KP923879532, Tfe, KP382683432 * Tff);
+		    Tgm = Tgk + Tgl;
+		    ThN = Tgk - Tgl;
+		    T2l = T2d + T2k;
+		    T2u = T2m - T2t;
+		    T2v = KP707106781 * (T2l + T2u);
+		    T6m = KP707106781 * (T2l - T2u);
+	       }
+	       {
+		    E T4I, T4J, Tc6, Tcd;
+		    T4I = T2k - T2d;
+		    T4J = T2m + T2t;
+		    T4K = KP707106781 * (T4I + T4J);
+		    T5P = KP707106781 * (T4J - T4I);
+		    Tc6 = FNMS(KP382683432, Tc5, KP923879532 * Tc2);
+		    Tcd = FMA(KP923879532, Tc9, KP382683432 * Tcc);
+		    Tce = Tc6 + Tcd;
+		    TdF = Tcd - Tc6;
+	       }
+	       {
+		    E TcB, TcC, T7j, T7k;
+		    TcB = FMA(KP923879532, Tc5, KP382683432 * Tc2);
+		    TcC = FNMS(KP382683432, Tc9, KP923879532 * Tcc);
+		    TcD = TcB + TcC;
+		    Tec = TcB - TcC;
+		    T7j = T2g + T2j;
+		    T7k = T2s + T2p;
+		    T7l = T7j - T7k;
+		    T9p = T7j + T7k;
+	       }
+	  }
+	  {
+	       E T1z, T1C, T1D, Tbg, TaQ, T4r, T4u, T7Q, Tbh, TaR, T1G, T3V, T1J, T3Y, T1K;
+	       E T7R, Tbe, Tbd, TaO, TaL, T1S, TfV, TfW, T41, T48, TaW, TaZ, T7L, T1Z, TfY;
+	       E TfZ, T4a, T4h, Tb3, Tb6, T7M;
+	       {
+		    E T1x, T1y, T1A, T1B;
+		    T1x = iio[-WS(ios, 63)];
+		    T1y = rio[WS(ios, 31)];
+		    T1z = T1x + T1y;
+		    T1A = rio[WS(ios, 15)];
+		    T1B = iio[-WS(ios, 47)];
+		    T1C = T1A + T1B;
+		    T1D = T1z + T1C;
+		    Tbg = T1x - T1y;
+		    TaQ = T1A - T1B;
+	       }
+	       {
+		    E T4p, T4q, T4s, T4t;
+		    T4p = iio[-WS(ios, 31)];
+		    T4q = rio[WS(ios, 63)];
+		    T4r = T4p - T4q;
+		    T4s = iio[-WS(ios, 15)];
+		    T4t = rio[WS(ios, 47)];
+		    T4u = T4s - T4t;
+		    T7Q = T4r + T4u;
+		    Tbh = T4s + T4t;
+		    TaR = T4q + T4p;
+	       }
+	       {
+		    E TaJ, TaK, TaM, TaN;
+		    {
+			 E T1E, T1F, T3T, T3U;
+			 T1E = rio[WS(ios, 7)];
+			 T1F = iio[-WS(ios, 39)];
+			 T1G = T1E + T1F;
+			 TaJ = T1E - T1F;
+			 T3T = iio[-WS(ios, 7)];
+			 T3U = rio[WS(ios, 39)];
+			 T3V = T3T - T3U;
+			 TaK = T3T + T3U;
+		    }
+		    {
+			 E T1H, T1I, T3W, T3X;
+			 T1H = iio[-WS(ios, 55)];
+			 T1I = rio[WS(ios, 23)];
+			 T1J = T1H + T1I;
+			 TaM = T1H - T1I;
+			 T3W = iio[-WS(ios, 23)];
+			 T3X = rio[WS(ios, 55)];
+			 T3Y = T3W - T3X;
+			 TaN = T3X + T3W;
+		    }
+		    T1K = T1G + T1J;
+		    T7R = T3V + T3Y;
+		    Tbe = TaK - TaJ;
+		    Tbd = TaM - TaN;
+		    TaO = TaM + TaN;
+		    TaL = TaJ + TaK;
+	       }
+	       {
+		    E T1O, TaX, T44, TaV, T1R, TaU, T47, TaY;
+		    {
+			 E T1M, T1N, T42, T43;
+			 T1M = rio[WS(ios, 3)];
+			 T1N = iio[-WS(ios, 35)];
+			 T1O = T1M + T1N;
+			 TaX = T1M - T1N;
+			 T42 = iio[-WS(ios, 3)];
+			 T43 = rio[WS(ios, 35)];
+			 T44 = T42 - T43;
+			 TaV = T42 + T43;
+		    }
+		    {
+			 E T1P, T1Q, T45, T46;
+			 T1P = rio[WS(ios, 19)];
+			 T1Q = iio[-WS(ios, 51)];
+			 T1R = T1P + T1Q;
+			 TaU = T1P - T1Q;
+			 T45 = iio[-WS(ios, 19)];
+			 T46 = rio[WS(ios, 51)];
+			 T47 = T45 - T46;
+			 TaY = T45 + T46;
+		    }
+		    T1S = T1O + T1R;
+		    TfV = TaV - TaU;
+		    TfW = TaX + TaY;
+		    T41 = T1O - T1R;
+		    T48 = T44 - T47;
+		    TaW = TaU + TaV;
+		    TaZ = TaX - TaY;
+		    T7L = T44 + T47;
+	       }
+	       {
+		    E T1V, Tb4, T4d, Tb2, T1Y, Tb1, T4g, Tb5;
+		    {
+			 E T1T, T1U, T4b, T4c;
+			 T1T = iio[-WS(ios, 59)];
+			 T1U = rio[WS(ios, 27)];
+			 T1V = T1T + T1U;
+			 Tb4 = T1T - T1U;
+			 T4b = iio[-WS(ios, 27)];
+			 T4c = rio[WS(ios, 59)];
+			 T4d = T4b - T4c;
+			 Tb2 = T4c + T4b;
+		    }
+		    {
+			 E T1W, T1X, T4e, T4f;
+			 T1W = rio[WS(ios, 11)];
+			 T1X = iio[-WS(ios, 43)];
+			 T1Y = T1W + T1X;
+			 Tb1 = T1W - T1X;
+			 T4e = iio[-WS(ios, 11)];
+			 T4f = rio[WS(ios, 43)];
+			 T4g = T4e - T4f;
+			 Tb5 = T4e + T4f;
+		    }
+		    T1Z = T1V + T1Y;
+		    TfY = Tb4 + Tb5;
+		    TfZ = Tb1 + Tb2;
+		    T4a = T1V - T1Y;
+		    T4h = T4d - T4g;
+		    Tb3 = Tb1 - Tb2;
+		    Tb6 = Tb4 - Tb5;
+		    T7M = T4g + T4d;
+	       }
+	       T1L = T1D + T1K;
+	       T20 = T1S + T1Z;
+	       T9c = T1L - T20;
+	       T9d = T7R + T7Q;
+	       T9e = T7L + T7M;
+	       T9f = T9d - T9e;
+	       {
+		    E T3S, T3Z, TfX, Tg0;
+		    T3S = T1z - T1C;
+		    T3Z = T3V - T3Y;
+		    T40 = T3S + T3Z;
+		    T66 = T3S - T3Z;
+		    TfX = FNMS(KP382683432, TfW, KP923879532 * TfV);
+		    Tg0 = FNMS(KP923879532, TfZ, KP382683432 * TfY);
+		    Tg1 = TfX + Tg0;
+		    Thu = Tg0 - TfX;
+	       }
+	       {
+		    E Tg6, Tg7, Tg3, Tg4;
+		    Tg6 = KP707106781 * (TaL + TaO);
+		    Tg7 = Tbg + Tbh;
+		    Tg8 = Tg6 + Tg7;
+		    Thv = Tg7 - Tg6;
+		    Tg3 = FMA(KP382683432, TfV, KP923879532 * TfW);
+		    Tg4 = FMA(KP923879532, TfY, KP382683432 * TfZ);
+		    Tg5 = Tg3 + Tg4;
+		    Thr = Tg3 - Tg4;
+	       }
+	       {
+		    E T4l, T4m, T49, T4i;
+		    T4l = T48 - T41;
+		    T4m = T4a + T4h;
+		    T4n = KP707106781 * (T4l + T4m);
+		    T67 = KP707106781 * (T4m - T4l);
+		    T49 = T41 + T48;
+		    T4i = T4a - T4h;
+		    T4j = KP707106781 * (T49 + T4i);
+		    T69 = KP707106781 * (T49 - T4i);
+	       }
+	       {
+		    E T4o, T4v, TaP, TaS;
+		    T4o = T1J - T1G;
+		    T4v = T4r - T4u;
+		    T4w = T4o + T4v;
+		    T6a = T4v - T4o;
+		    TaP = KP707106781 * (TaL - TaO);
+		    TaS = TaQ - TaR;
+		    TaT = TaP + TaS;
+		    TdW = TaS - TaP;
+	       }
+	       {
+		    E Tb0, Tb7, TfS, TfT;
+		    Tb0 = FMA(KP923879532, TaW, KP382683432 * TaZ);
+		    Tb7 = FNMS(KP382683432, Tb6, KP923879532 * Tb3);
+		    Tb8 = Tb0 + Tb7;
+		    TdZ = Tb0 - Tb7;
+		    TfS = KP707106781 * (Tbe + Tbd);
+		    TfT = TaQ + TaR;
+		    TfU = TfS - TfT;
+		    Ths = TfS + TfT;
+	       }
+	       {
+		    E T7K, T7N, T7P, T7S;
+		    T7K = T1D - T1K;
+		    T7N = T7L - T7M;
+		    T7O = T7K + T7N;
+		    T8y = T7K - T7N;
+		    T7P = T1Z - T1S;
+		    T7S = T7Q - T7R;
+		    T7T = T7P + T7S;
+		    T8z = T7S - T7P;
+	       }
+	       {
+		    E Tba, Tbb, Tbf, Tbi;
+		    Tba = FNMS(KP382683432, TaW, KP923879532 * TaZ);
+		    Tbb = FMA(KP923879532, Tb6, KP382683432 * Tb3);
+		    Tbc = Tba + Tbb;
+		    TdX = Tbb - Tba;
+		    Tbf = KP707106781 * (Tbd - Tbe);
+		    Tbi = Tbg - Tbh;
+		    Tbj = Tbf + Tbi;
+		    Te0 = Tbi - Tbf;
+	       }
+	  }
+	  {
+	       E T14, T17, T18, Tax, Tas, T3K, T3N, T7F, Tay, Tat, T1b, T3e, T1e, T3h, T1f;
+	       E T7G, TaB, TaA, Taq, Tan, T1n, TfC, TfD, T3k, T3r, Ta8, Tab, T7A, T1u, TfF;
+	       E TfG, T3t, T3A, Taf, Tai, T7B;
+	       {
+		    E T12, T13, T15, T16;
+		    T12 = rio[WS(ios, 1)];
+		    T13 = iio[-WS(ios, 33)];
+		    T14 = T12 + T13;
+		    T15 = rio[WS(ios, 17)];
+		    T16 = iio[-WS(ios, 49)];
+		    T17 = T15 + T16;
+		    T18 = T14 + T17;
+		    Tax = T15 - T16;
+		    Tas = T12 - T13;
+	       }
+	       {
+		    E T3I, T3J, T3L, T3M;
+		    T3I = iio[-WS(ios, 1)];
+		    T3J = rio[WS(ios, 33)];
+		    T3K = T3I - T3J;
+		    T3L = iio[-WS(ios, 17)];
+		    T3M = rio[WS(ios, 49)];
+		    T3N = T3L - T3M;
+		    T7F = T3K + T3N;
+		    Tay = T3I + T3J;
+		    Tat = T3L + T3M;
+	       }
+	       {
+		    E Tap, Tao, Tal, Tam;
+		    {
+			 E T19, T1a, T3c, T3d;
+			 T19 = rio[WS(ios, 9)];
+			 T1a = iio[-WS(ios, 41)];
+			 T1b = T19 + T1a;
+			 Tap = T19 - T1a;
+			 T3c = iio[-WS(ios, 9)];
+			 T3d = rio[WS(ios, 41)];
+			 T3e = T3c - T3d;
+			 Tao = T3c + T3d;
+		    }
+		    {
+			 E T1c, T1d, T3f, T3g;
+			 T1c = iio[-WS(ios, 57)];
+			 T1d = rio[WS(ios, 25)];
+			 T1e = T1c + T1d;
+			 Tal = T1c - T1d;
+			 T3f = iio[-WS(ios, 25)];
+			 T3g = rio[WS(ios, 57)];
+			 T3h = T3f - T3g;
+			 Tam = T3g + T3f;
+		    }
+		    T1f = T1b + T1e;
+		    T7G = T3e + T3h;
+		    TaB = Tal + Tam;
+		    TaA = Tap + Tao;
+		    Taq = Tao - Tap;
+		    Tan = Tal - Tam;
+	       }
+	       {
+		    E T1j, Ta6, T3n, Taa, T1m, Ta9, T3q, Ta7;
+		    {
+			 E T1h, T1i, T3l, T3m;
+			 T1h = rio[WS(ios, 5)];
+			 T1i = iio[-WS(ios, 37)];
+			 T1j = T1h + T1i;
+			 Ta6 = T1h - T1i;
+			 T3l = iio[-WS(ios, 5)];
+			 T3m = rio[WS(ios, 37)];
+			 T3n = T3l - T3m;
+			 Taa = T3l + T3m;
+		    }
+		    {
+			 E T1k, T1l, T3o, T3p;
+			 T1k = rio[WS(ios, 21)];
+			 T1l = iio[-WS(ios, 53)];
+			 T1m = T1k + T1l;
+			 Ta9 = T1k - T1l;
+			 T3o = iio[-WS(ios, 21)];
+			 T3p = rio[WS(ios, 53)];
+			 T3q = T3o - T3p;
+			 Ta7 = T3o + T3p;
+		    }
+		    T1n = T1j + T1m;
+		    TfC = Taa - Ta9;
+		    TfD = Ta6 + Ta7;
+		    T3k = T1j - T1m;
+		    T3r = T3n - T3q;
+		    Ta8 = Ta6 - Ta7;
+		    Tab = Ta9 + Taa;
+		    T7A = T3n + T3q;
+	       }
+	       {
+		    E T1q, Tad, T3w, Tah, T1t, Tag, T3z, Tae;
+		    {
+			 E T1o, T1p, T3u, T3v;
+			 T1o = iio[-WS(ios, 61)];
+			 T1p = rio[WS(ios, 29)];
+			 T1q = T1o + T1p;
+			 Tad = T1o - T1p;
+			 T3u = iio[-WS(ios, 29)];
+			 T3v = rio[WS(ios, 61)];
+			 T3w = T3u - T3v;
+			 Tah = T3v + T3u;
+		    }
+		    {
+			 E T1r, T1s, T3x, T3y;
+			 T1r = rio[WS(ios, 13)];
+			 T1s = iio[-WS(ios, 45)];
+			 T1t = T1r + T1s;
+			 Tag = T1r - T1s;
+			 T3x = iio[-WS(ios, 13)];
+			 T3y = rio[WS(ios, 45)];
+			 T3z = T3x - T3y;
+			 Tae = T3x + T3y;
+		    }
+		    T1u = T1q + T1t;
+		    TfF = Tad + Tae;
+		    TfG = Tag + Tah;
+		    T3t = T1q - T1t;
+		    T3A = T3w - T3z;
+		    Taf = Tad - Tae;
+		    Tai = Tag - Tah;
+		    T7B = T3z + T3w;
+	       }
+	       T1g = T18 + T1f;
+	       T1v = T1n + T1u;
+	       T97 = T1g - T1v;
+	       T98 = T7G + T7F;
+	       T99 = T7A + T7B;
+	       T9a = T98 - T99;
+	       {
+		    E T3b, T3i, TfE, TfH;
+		    T3b = T14 - T17;
+		    T3i = T3e - T3h;
+		    T3j = T3b + T3i;
+		    T5Z = T3b - T3i;
+		    TfE = FNMS(KP382683432, TfD, KP923879532 * TfC);
+		    TfH = FNMS(KP923879532, TfG, KP382683432 * TfF);
+		    TfI = TfE + TfH;
+		    Thk = TfH - TfE;
+	       }
+	       {
+		    E TfN, TfO, TfK, TfL;
+		    TfN = KP707106781 * (TaA + TaB);
+		    TfO = Tas + Tat;
+		    TfP = TfN + TfO;
+		    Thl = TfO - TfN;
+		    TfK = FMA(KP382683432, TfC, KP923879532 * TfD);
+		    TfL = FMA(KP923879532, TfF, KP382683432 * TfG);
+		    TfM = TfK + TfL;
+		    Tho = TfK - TfL;
+	       }
+	       {
+		    E T3E, T3F, T3s, T3B;
+		    T3E = T3r - T3k;
+		    T3F = T3t + T3A;
+		    T3G = KP707106781 * (T3E + T3F);
+		    T60 = KP707106781 * (T3F - T3E);
+		    T3s = T3k + T3r;
+		    T3B = T3t - T3A;
+		    T3C = KP707106781 * (T3s + T3B);
+		    T62 = KP707106781 * (T3s - T3B);
+	       }
+	       {
+		    E T3H, T3O, Tac, Taj;
+		    T3H = T1e - T1b;
+		    T3O = T3K - T3N;
+		    T3P = T3H + T3O;
+		    T63 = T3O - T3H;
+		    Tac = FNMS(KP382683432, Tab, KP923879532 * Ta8);
+		    Taj = FMA(KP923879532, Taf, KP382683432 * Tai);
+		    Tak = Tac + Taj;
+		    TdQ = Taj - Tac;
+	       }
+	       {
+		    E Tar, Tau, Tfz, TfA;
+		    Tar = KP707106781 * (Tan - Taq);
+		    Tau = Tas - Tat;
+		    Tav = Tar + Tau;
+		    TdT = Tau - Tar;
+		    Tfz = Tay - Tax;
+		    TfA = KP707106781 * (Taq + Tan);
+		    TfB = Tfz + TfA;
+		    Thn = Tfz - TfA;
+	       }
+	       {
+		    E T7z, T7C, T7E, T7H;
+		    T7z = T18 - T1f;
+		    T7C = T7A - T7B;
+		    T7D = T7z + T7C;
+		    T8v = T7z - T7C;
+		    T7E = T1u - T1n;
+		    T7H = T7F - T7G;
+		    T7I = T7E + T7H;
+		    T8w = T7H - T7E;
+	       }
+	       {
+		    E Taz, TaC, TaE, TaF;
+		    Taz = Tax + Tay;
+		    TaC = KP707106781 * (TaA - TaB);
+		    TaD = Taz + TaC;
+		    TdP = Taz - TaC;
+		    TaE = FMA(KP923879532, Tab, KP382683432 * Ta8);
+		    TaF = FNMS(KP382683432, Taf, KP923879532 * Tai);
+		    TaG = TaE + TaF;
+		    TdS = TaE - TaF;
+	       }
+	  }
+	  {
+	       E T11, T9K, T9T, Ta2, T22, T9Q, T9N, Ta3;
+	       {
+		    E Tv, T10, T9R, T9S;
+		    Tv = Tf + Tu;
+		    T10 = TK + TZ;
+		    T11 = Tv + T10;
+		    T9K = Tv - T10;
+		    T9R = T9p + T9o;
+		    T9S = T93 + T94;
+		    T9T = T9R - T9S;
+		    Ta2 = T9S + T9R;
+	       }
+	       {
+		    E T1w, T21, T9L, T9M;
+		    T1w = T1g + T1v;
+		    T21 = T1L + T20;
+		    T22 = T1w + T21;
+		    T9Q = T21 - T1w;
+		    T9L = T99 + T98;
+		    T9M = T9e + T9d;
+		    T9N = T9L - T9M;
+		    Ta3 = T9L + T9M;
+	       }
+	       rio[0] = T11 + T22;
+	       iio[-WS(ios, 63)] = Ta3 + Ta2;
+	       {
+		    E T9O, T9U, T9J, T9P;
+		    T9O = T9K + T9N;
+		    T9U = T9Q + T9T;
+		    T9J = W[94];
+		    T9P = W[95];
+		    rio[WS(ios, 48)] = FNMS(T9P, T9U, T9J * T9O);
+		    iio[-WS(ios, 15)] = FMA(T9P, T9O, T9J * T9U);
+	       }
+	       {
+		    E T9W, T9Y, T9V, T9X;
+		    T9W = T9K - T9N;
+		    T9Y = T9T - T9Q;
+		    T9V = W[30];
+		    T9X = W[31];
+		    rio[WS(ios, 16)] = FNMS(T9X, T9Y, T9V * T9W);
+		    iio[-WS(ios, 47)] = FMA(T9X, T9W, T9V * T9Y);
+	       }
+	       {
+		    E Ta0, Ta4, T9Z, Ta1;
+		    Ta0 = T11 - T22;
+		    Ta4 = Ta2 - Ta3;
+		    T9Z = W[62];
+		    Ta1 = W[63];
+		    rio[WS(ios, 32)] = FNMS(Ta1, Ta4, T9Z * Ta0);
+		    iio[-WS(ios, 31)] = FMA(Ta1, Ta0, T9Z * Ta4);
+	       }
+	  }
+	  {
+	       E T96, T9y, T9r, T9D, T9h, T9C, T9m, T9z;
+	       {
+		    E T92, T95, T9n, T9q;
+		    T92 = Tf - Tu;
+		    T95 = T93 - T94;
+		    T96 = T92 + T95;
+		    T9y = T92 - T95;
+		    T9n = TZ - TK;
+		    T9q = T9o - T9p;
+		    T9r = T9n + T9q;
+		    T9D = T9q - T9n;
+	       }
+	       {
+		    E T9b, T9g, T9k, T9l;
+		    T9b = T97 + T9a;
+		    T9g = T9c - T9f;
+		    T9h = KP707106781 * (T9b + T9g);
+		    T9C = KP707106781 * (T9b - T9g);
+		    T9k = T9a - T97;
+		    T9l = T9c + T9f;
+		    T9m = KP707106781 * (T9k + T9l);
+		    T9z = KP707106781 * (T9l - T9k);
+	       }
+	       {
+		    E T9i, T9s, T91, T9j;
+		    T9i = T96 + T9h;
+		    T9s = T9m + T9r;
+		    T91 = W[110];
+		    T9j = W[111];
+		    rio[WS(ios, 56)] = FNMS(T9j, T9s, T91 * T9i);
+		    iio[-WS(ios, 7)] = FMA(T9j, T9i, T91 * T9s);
+	       }
+	       {
+		    E T9G, T9I, T9F, T9H;
+		    T9G = T9y - T9z;
+		    T9I = T9D - T9C;
+		    T9F = W[78];
+		    T9H = W[79];
+		    rio[WS(ios, 40)] = FNMS(T9H, T9I, T9F * T9G);
+		    iio[-WS(ios, 23)] = FMA(T9H, T9G, T9F * T9I);
+	       }
+	       {
+		    E T9u, T9w, T9t, T9v;
+		    T9u = T96 - T9h;
+		    T9w = T9r - T9m;
+		    T9t = W[46];
+		    T9v = W[47];
+		    rio[WS(ios, 24)] = FNMS(T9v, T9w, T9t * T9u);
+		    iio[-WS(ios, 39)] = FMA(T9v, T9u, T9t * T9w);
+	       }
+	       {
+		    E T9A, T9E, T9x, T9B;
+		    T9A = T9y + T9z;
+		    T9E = T9C + T9D;
+		    T9x = W[14];
+		    T9B = W[15];
+		    rio[WS(ios, 8)] = FNMS(T9B, T9E, T9x * T9A);
+		    iio[-WS(ios, 55)] = FMA(T9B, T9A, T9x * T9E);
+	       }
+	  }
+	  {
+	       E T8u, T8Q, T8J, T8V, T8B, T8U, T8G, T8R;
+	       {
+		    E T8s, T8t, T8H, T8I;
+		    T8s = T7i - T7l;
+		    T8t = KP707106781 * (T82 - T81);
+		    T8u = T8s + T8t;
+		    T8Q = T8s - T8t;
+		    T8H = KP707106781 * (T7r - T7w);
+		    T8I = T87 - T84;
+		    T8J = T8H + T8I;
+		    T8V = T8I - T8H;
+	       }
+	       {
+		    E T8x, T8A, T8E, T8F;
+		    T8x = FNMS(KP382683432, T8w, KP923879532 * T8v);
+		    T8A = FMA(KP923879532, T8y, KP382683432 * T8z);
+		    T8B = T8x + T8A;
+		    T8U = T8A - T8x;
+		    T8E = FMA(KP382683432, T8v, KP923879532 * T8w);
+		    T8F = FNMS(KP382683432, T8y, KP923879532 * T8z);
+		    T8G = T8E + T8F;
+		    T8R = T8E - T8F;
+	       }
+	       {
+		    E T8C, T8K, T8r, T8D;
+		    T8C = T8u + T8B;
+		    T8K = T8G + T8J;
+		    T8r = W[6];
+		    T8D = W[7];
+		    rio[WS(ios, 4)] = FNMS(T8D, T8K, T8r * T8C);
+		    iio[-WS(ios, 59)] = FMA(T8D, T8C, T8r * T8K);
+	       }
+	       {
+		    E T8Y, T90, T8X, T8Z;
+		    T8Y = T8Q - T8R;
+		    T90 = T8V - T8U;
+		    T8X = W[38];
+		    T8Z = W[39];
+		    rio[WS(ios, 20)] = FNMS(T8Z, T90, T8X * T8Y);
+		    iio[-WS(ios, 43)] = FMA(T8Z, T8Y, T8X * T90);
+	       }
+	       {
+		    E T8M, T8O, T8L, T8N;
+		    T8M = T8u - T8B;
+		    T8O = T8J - T8G;
+		    T8L = W[70];
+		    T8N = W[71];
+		    rio[WS(ios, 36)] = FNMS(T8N, T8O, T8L * T8M);
+		    iio[-WS(ios, 27)] = FMA(T8N, T8M, T8L * T8O);
+	       }
+	       {
+		    E T8S, T8W, T8P, T8T;
+		    T8S = T8Q + T8R;
+		    T8W = T8U + T8V;
+		    T8P = W[102];
+		    T8T = W[103];
+		    rio[WS(ios, 52)] = FNMS(T8T, T8W, T8P * T8S);
+		    iio[-WS(ios, 11)] = FMA(T8T, T8S, T8P * T8W);
+	       }
+	  }
+	  {
+	       E T7y, T8g, T89, T8l, T7V, T8k, T80, T8h;
+	       {
+		    E T7m, T7x, T83, T88;
+		    T7m = T7i + T7l;
+		    T7x = KP707106781 * (T7r + T7w);
+		    T7y = T7m + T7x;
+		    T8g = T7m - T7x;
+		    T83 = KP707106781 * (T81 + T82);
+		    T88 = T84 + T87;
+		    T89 = T83 + T88;
+		    T8l = T88 - T83;
+	       }
+	       {
+		    E T7J, T7U, T7Y, T7Z;
+		    T7J = FMA(KP923879532, T7D, KP382683432 * T7I);
+		    T7U = FNMS(KP382683432, T7T, KP923879532 * T7O);
+		    T7V = T7J + T7U;
+		    T8k = T7J - T7U;
+		    T7Y = FNMS(KP382683432, T7D, KP923879532 * T7I);
+		    T7Z = FMA(KP382683432, T7O, KP923879532 * T7T);
+		    T80 = T7Y + T7Z;
+		    T8h = T7Z - T7Y;
+	       }
+	       {
+		    E T7W, T8a, T7h, T7X;
+		    T7W = T7y + T7V;
+		    T8a = T80 + T89;
+		    T7h = W[118];
+		    T7X = W[119];
+		    rio[WS(ios, 60)] = FNMS(T7X, T8a, T7h * T7W);
+		    iio[-WS(ios, 3)] = FMA(T7X, T7W, T7h * T8a);
+	       }
+	       {
+		    E T8o, T8q, T8n, T8p;
+		    T8o = T8g - T8h;
+		    T8q = T8l - T8k;
+		    T8n = W[86];
+		    T8p = W[87];
+		    rio[WS(ios, 44)] = FNMS(T8p, T8q, T8n * T8o);
+		    iio[-WS(ios, 19)] = FMA(T8p, T8o, T8n * T8q);
+	       }
+	       {
+		    E T8c, T8e, T8b, T8d;
+		    T8c = T7y - T7V;
+		    T8e = T89 - T80;
+		    T8b = W[54];
+		    T8d = W[55];
+		    rio[WS(ios, 28)] = FNMS(T8d, T8e, T8b * T8c);
+		    iio[-WS(ios, 35)] = FMA(T8d, T8c, T8b * T8e);
+	       }
+	       {
+		    E T8i, T8m, T8f, T8j;
+		    T8i = T8g + T8h;
+		    T8m = T8k + T8l;
+		    T8f = W[22];
+		    T8j = W[23];
+		    rio[WS(ios, 12)] = FNMS(T8j, T8m, T8f * T8i);
+		    iio[-WS(ios, 51)] = FMA(T8j, T8i, T8f * T8m);
+	       }
+	  }
+	  {
+	       E T6K, T76, T6Z, T7b, T6R, T7a, T6W, T77;
+	       {
+		    E T6I, T6J, T6X, T6Y;
+		    T6I = T5O - T5P;
+		    T6J = T6j - T6k;
+		    T6K = T6I + T6J;
+		    T76 = T6I - T6J;
+		    T6X = T5W - T5T;
+		    T6Y = T6n - T6m;
+		    T6Z = T6X + T6Y;
+		    T7b = T6Y - T6X;
+		    {
+			 E T6N, T6U, T6Q, T6V;
+			 {
+			      E T6L, T6M, T6O, T6P;
+			      T6L = T5Z - T60;
+			      T6M = T63 - T62;
+			      T6N = FMA(KP831469612, T6L, KP555570233 * T6M);
+			      T6U = FNMS(KP555570233, T6L, KP831469612 * T6M);
+			      T6O = T66 - T67;
+			      T6P = T6a - T69;
+			      T6Q = FNMS(KP555570233, T6P, KP831469612 * T6O);
+			      T6V = FMA(KP555570233, T6O, KP831469612 * T6P);
+			 }
+			 T6R = T6N + T6Q;
+			 T7a = T6N - T6Q;
+			 T6W = T6U + T6V;
+			 T77 = T6V - T6U;
+		    }
+	       }
+	       {
+		    E T6S, T70, T6H, T6T;
+		    T6S = T6K + T6R;
+		    T70 = T6W + T6Z;
+		    T6H = W[114];
+		    T6T = W[115];
+		    rio[WS(ios, 58)] = FNMS(T6T, T70, T6H * T6S);
+		    iio[-WS(ios, 5)] = FMA(T6T, T6S, T6H * T70);
+	       }
+	       {
+		    E T7e, T7g, T7d, T7f;
+		    T7e = T76 - T77;
+		    T7g = T7b - T7a;
+		    T7d = W[82];
+		    T7f = W[83];
+		    rio[WS(ios, 42)] = FNMS(T7f, T7g, T7d * T7e);
+		    iio[-WS(ios, 21)] = FMA(T7f, T7e, T7d * T7g);
+	       }
+	       {
+		    E T72, T74, T71, T73;
+		    T72 = T6K - T6R;
+		    T74 = T6Z - T6W;
+		    T71 = W[50];
+		    T73 = W[51];
+		    rio[WS(ios, 26)] = FNMS(T73, T74, T71 * T72);
+		    iio[-WS(ios, 37)] = FMA(T73, T72, T71 * T74);
+	       }
+	       {
+		    E T78, T7c, T75, T79;
+		    T78 = T76 + T77;
+		    T7c = T7a + T7b;
+		    T75 = W[18];
+		    T79 = W[19];
+		    rio[WS(ios, 10)] = FNMS(T79, T7c, T75 * T78);
+		    iio[-WS(ios, 53)] = FMA(T79, T78, T75 * T7c);
+	       }
+	  }
+	  {
+	       E T3a, T52, T4V, T57, T4z, T56, T4E, T53;
+	       {
+		    E T2w, T39, T4H, T4U;
+		    T2w = T2c + T2v;
+		    T39 = T2P + T38;
+		    T3a = T2w + T39;
+		    T52 = T2w - T39;
+		    T4H = T4F + T4G;
+		    T4U = T4K + T4T;
+		    T4V = T4H + T4U;
+		    T57 = T4U - T4H;
+		    {
+			 E T3R, T4C, T4y, T4D;
+			 {
+			      E T3D, T3Q, T4k, T4x;
+			      T3D = T3j + T3C;
+			      T3Q = T3G + T3P;
+			      T3R = FMA(KP980785280, T3D, KP195090322 * T3Q);
+			      T4C = FNMS(KP195090322, T3D, KP980785280 * T3Q);
+			      T4k = T40 + T4j;
+			      T4x = T4n + T4w;
+			      T4y = FNMS(KP195090322, T4x, KP980785280 * T4k);
+			      T4D = FMA(KP195090322, T4k, KP980785280 * T4x);
+			 }
+			 T4z = T3R + T4y;
+			 T56 = T3R - T4y;
+			 T4E = T4C + T4D;
+			 T53 = T4D - T4C;
+		    }
+	       }
+	       {
+		    E T4A, T4W, T23, T4B;
+		    T4A = T3a + T4z;
+		    T4W = T4E + T4V;
+		    T23 = W[122];
+		    T4B = W[123];
+		    rio[WS(ios, 62)] = FNMS(T4B, T4W, T23 * T4A);
+		    iio[-WS(ios, 1)] = FMA(T4B, T4A, T23 * T4W);
+	       }
+	       {
+		    E T5a, T5c, T59, T5b;
+		    T5a = T52 - T53;
+		    T5c = T57 - T56;
+		    T59 = W[90];
+		    T5b = W[91];
+		    rio[WS(ios, 46)] = FNMS(T5b, T5c, T59 * T5a);
+		    iio[-WS(ios, 17)] = FMA(T5b, T5a, T59 * T5c);
+	       }
+	       {
+		    E T4Y, T50, T4X, T4Z;
+		    T4Y = T3a - T4z;
+		    T50 = T4V - T4E;
+		    T4X = W[58];
+		    T4Z = W[59];
+		    rio[WS(ios, 30)] = FNMS(T4Z, T50, T4X * T4Y);
+		    iio[-WS(ios, 33)] = FMA(T4Z, T4Y, T4X * T50);
+	       }
+	       {
+		    E T54, T58, T51, T55;
+		    T54 = T52 + T53;
+		    T58 = T56 + T57;
+		    T51 = W[26];
+		    T55 = W[27];
+		    rio[WS(ios, 14)] = FNMS(T55, T58, T51 * T54);
+		    iio[-WS(ios, 49)] = FMA(T55, T54, T51 * T58);
+	       }
+	  }
+	  {
+	       E T5g, T5C, T5v, T5H, T5n, T5G, T5s, T5D;
+	       {
+		    E T5e, T5f, T5t, T5u;
+		    T5e = T2c - T2v;
+		    T5f = T4G - T4F;
+		    T5g = T5e + T5f;
+		    T5C = T5e - T5f;
+		    T5t = T2P - T38;
+		    T5u = T4T - T4K;
+		    T5v = T5t + T5u;
+		    T5H = T5u - T5t;
+		    {
+			 E T5j, T5q, T5m, T5r;
+			 {
+			      E T5h, T5i, T5k, T5l;
+			      T5h = T3j - T3C;
+			      T5i = T3P - T3G;
+			      T5j = FNMS(KP555570233, T5i, KP831469612 * T5h);
+			      T5q = FMA(KP555570233, T5h, KP831469612 * T5i);
+			      T5k = T40 - T4j;
+			      T5l = T4w - T4n;
+			      T5m = FMA(KP831469612, T5k, KP555570233 * T5l);
+			      T5r = FNMS(KP555570233, T5k, KP831469612 * T5l);
+			 }
+			 T5n = T5j + T5m;
+			 T5G = T5m - T5j;
+			 T5s = T5q + T5r;
+			 T5D = T5q - T5r;
+		    }
+	       }
+	       {
+		    E T5o, T5w, T5d, T5p;
+		    T5o = T5g + T5n;
+		    T5w = T5s + T5v;
+		    T5d = W[10];
+		    T5p = W[11];
+		    rio[WS(ios, 6)] = FNMS(T5p, T5w, T5d * T5o);
+		    iio[-WS(ios, 57)] = FMA(T5p, T5o, T5d * T5w);
+	       }
+	       {
+		    E T5K, T5M, T5J, T5L;
+		    T5K = T5C - T5D;
+		    T5M = T5H - T5G;
+		    T5J = W[42];
+		    T5L = W[43];
+		    rio[WS(ios, 22)] = FNMS(T5L, T5M, T5J * T5K);
+		    iio[-WS(ios, 41)] = FMA(T5L, T5K, T5J * T5M);
+	       }
+	       {
+		    E T5y, T5A, T5x, T5z;
+		    T5y = T5g - T5n;
+		    T5A = T5v - T5s;
+		    T5x = W[74];
+		    T5z = W[75];
+		    rio[WS(ios, 38)] = FNMS(T5z, T5A, T5x * T5y);
+		    iio[-WS(ios, 25)] = FMA(T5z, T5y, T5x * T5A);
+	       }
+	       {
+		    E T5E, T5I, T5B, T5F;
+		    T5E = T5C + T5D;
+		    T5I = T5G + T5H;
+		    T5B = W[106];
+		    T5F = W[107];
+		    rio[WS(ios, 54)] = FNMS(T5F, T5I, T5B * T5E);
+		    iio[-WS(ios, 9)] = FMA(T5F, T5E, T5B * T5I);
+	       }
+	  }
+	  {
+	       E T5Y, T6w, T6p, T6B, T6d, T6A, T6i, T6x;
+	       {
+		    E T5Q, T5X, T6l, T6o;
+		    T5Q = T5O + T5P;
+		    T5X = T5T + T5W;
+		    T5Y = T5Q + T5X;
+		    T6w = T5Q - T5X;
+		    T6l = T6j + T6k;
+		    T6o = T6m + T6n;
+		    T6p = T6l + T6o;
+		    T6B = T6o - T6l;
+		    {
+			 E T65, T6g, T6c, T6h;
+			 {
+			      E T61, T64, T68, T6b;
+			      T61 = T5Z + T60;
+			      T64 = T62 + T63;
+			      T65 = FNMS(KP195090322, T64, KP980785280 * T61);
+			      T6g = FMA(KP195090322, T61, KP980785280 * T64);
+			      T68 = T66 + T67;
+			      T6b = T69 + T6a;
+			      T6c = FMA(KP980785280, T68, KP195090322 * T6b);
+			      T6h = FNMS(KP195090322, T68, KP980785280 * T6b);
+			 }
+			 T6d = T65 + T6c;
+			 T6A = T6c - T65;
+			 T6i = T6g + T6h;
+			 T6x = T6g - T6h;
+		    }
+	       }
+	       {
+		    E T6e, T6q, T5N, T6f;
+		    T6e = T5Y + T6d;
+		    T6q = T6i + T6p;
+		    T5N = W[2];
+		    T6f = W[3];
+		    rio[WS(ios, 2)] = FNMS(T6f, T6q, T5N * T6e);
+		    iio[-WS(ios, 61)] = FMA(T6f, T6e, T5N * T6q);
+	       }
+	       {
+		    E T6E, T6G, T6D, T6F;
+		    T6E = T6w - T6x;
+		    T6G = T6B - T6A;
+		    T6D = W[34];
+		    T6F = W[35];
+		    rio[WS(ios, 18)] = FNMS(T6F, T6G, T6D * T6E);
+		    iio[-WS(ios, 45)] = FMA(T6F, T6E, T6D * T6G);
+	       }
+	       {
+		    E T6s, T6u, T6r, T6t;
+		    T6s = T5Y - T6d;
+		    T6u = T6p - T6i;
+		    T6r = W[66];
+		    T6t = W[67];
+		    rio[WS(ios, 34)] = FNMS(T6t, T6u, T6r * T6s);
+		    iio[-WS(ios, 29)] = FMA(T6t, T6s, T6r * T6u);
+	       }
+	       {
+		    E T6y, T6C, T6v, T6z;
+		    T6y = T6w + T6x;
+		    T6C = T6A + T6B;
+		    T6v = W[98];
+		    T6z = W[99];
+		    rio[WS(ios, 50)] = FNMS(T6z, T6C, T6v * T6y);
+		    iio[-WS(ios, 13)] = FMA(T6z, T6y, T6v * T6C);
+	       }
+	  }
+	  {
+	       E TdO, Tf1, Teq, TeH, Tef, TeW, Ten, TeM, Te3, Ter, Te8, Tem, TeE, Tf0, TeP;
+	       E TeX;
+	       {
+		    E TdG, TeG, TdN, TeF, TdJ, TdM;
+		    TdG = TdE + TdF;
+		    TeG = Ted - Tec;
+		    TdJ = FNMS(KP555570233, TdI, KP831469612 * TdH);
+		    TdM = FMA(KP831469612, TdK, KP555570233 * TdL);
+		    TdN = TdJ + TdM;
+		    TeF = TdM - TdJ;
+		    TdO = TdG + TdN;
+		    Tf1 = TeG - TeF;
+		    Teq = TdG - TdN;
+		    TeH = TeF + TeG;
+	       }
+	       {
+		    E Tee, TeK, Teb, TeL, Te9, Tea;
+		    Tee = Tec + Ted;
+		    TeK = TdE - TdF;
+		    Te9 = FMA(KP555570233, TdH, KP831469612 * TdI);
+		    Tea = FNMS(KP555570233, TdK, KP831469612 * TdL);
+		    Teb = Te9 + Tea;
+		    TeL = Te9 - Tea;
+		    Tef = Teb + Tee;
+		    TeW = TeK - TeL;
+		    Ten = Tee - Teb;
+		    TeM = TeK + TeL;
+	       }
+	       {
+		    E TdV, Te6, Te2, Te7;
+		    {
+			 E TdR, TdU, TdY, Te1;
+			 TdR = TdP + TdQ;
+			 TdU = TdS + TdT;
+			 TdV = FNMS(KP290284677, TdU, KP956940335 * TdR);
+			 Te6 = FMA(KP290284677, TdR, KP956940335 * TdU);
+			 TdY = TdW + TdX;
+			 Te1 = TdZ + Te0;
+			 Te2 = FMA(KP956940335, TdY, KP290284677 * Te1);
+			 Te7 = FNMS(KP290284677, TdY, KP956940335 * Te1);
+		    }
+		    Te3 = TdV + Te2;
+		    Ter = Te6 - Te7;
+		    Te8 = Te6 + Te7;
+		    Tem = Te2 - TdV;
+	       }
+	       {
+		    E TeA, TeN, TeD, TeO;
+		    {
+			 E Tey, Tez, TeB, TeC;
+			 Tey = TdT - TdS;
+			 Tez = TdP - TdQ;
+			 TeA = FNMS(KP471396736, Tez, KP881921264 * Tey);
+			 TeN = FMA(KP881921264, Tez, KP471396736 * Tey);
+			 TeB = TdW - TdX;
+			 TeC = Te0 - TdZ;
+			 TeD = FMA(KP471396736, TeB, KP881921264 * TeC);
+			 TeO = FNMS(KP471396736, TeC, KP881921264 * TeB);
+		    }
+		    TeE = TeA + TeD;
+		    Tf0 = TeN - TeO;
+		    TeP = TeN + TeO;
+		    TeX = TeD - TeA;
+	       }
+	       {
+		    E Te4, Teg, TdD, Te5;
+		    Te4 = TdO + Te3;
+		    Teg = Te8 + Tef;
+		    TdD = W[120];
+		    Te5 = W[121];
+		    iio[-WS(ios, 2)] = FMA(TdD, Te4, Te5 * Teg);
+		    rio[WS(ios, 61)] = FNMS(Te5, Te4, TdD * Teg);
+	       }
+	       {
+		    E TeY, Tf2, TeV, TeZ;
+		    TeY = TeW + TeX;
+		    Tf2 = Tf0 + Tf1;
+		    TeV = W[104];
+		    TeZ = W[105];
+		    iio[-WS(ios, 10)] = FMA(TeV, TeY, TeZ * Tf2);
+		    rio[WS(ios, 53)] = FNMS(TeZ, TeY, TeV * Tf2);
+	       }
+	       {
+		    E Tf4, Tf6, Tf3, Tf5;
+		    Tf4 = Tf1 - Tf0;
+		    Tf6 = TeW - TeX;
+		    Tf3 = W[40];
+		    Tf5 = W[41];
+		    rio[WS(ios, 21)] = FNMS(Tf5, Tf6, Tf3 * Tf4);
+		    iio[-WS(ios, 42)] = FMA(Tf3, Tf6, Tf5 * Tf4);
+	       }
+	       {
+		    E Tei, Tek, Teh, Tej;
+		    Tei = Tef - Te8;
+		    Tek = TdO - Te3;
+		    Teh = W[56];
+		    Tej = W[57];
+		    rio[WS(ios, 29)] = FNMS(Tej, Tek, Teh * Tei);
+		    iio[-WS(ios, 34)] = FMA(Teh, Tek, Tej * Tei);
+	       }
+	       {
+		    E Teo, Tes, Tel, Tep;
+		    Teo = Tem + Ten;
+		    Tes = Teq + Ter;
+		    Tel = W[24];
+		    Tep = W[25];
+		    rio[WS(ios, 13)] = FNMS(Tep, Tes, Tel * Teo);
+		    iio[-WS(ios, 50)] = FMA(Tel, Tes, Tep * Teo);
+	       }
+	       {
+		    E TeI, TeQ, Tex, TeJ;
+		    TeI = TeE + TeH;
+		    TeQ = TeM + TeP;
+		    Tex = W[8];
+		    TeJ = W[9];
+		    rio[WS(ios, 5)] = FNMS(TeJ, TeQ, Tex * TeI);
+		    iio[-WS(ios, 58)] = FMA(Tex, TeQ, TeJ * TeI);
+	       }
+	       {
+		    E TeS, TeU, TeR, TeT;
+		    TeS = TeM - TeP;
+		    TeU = TeH - TeE;
+		    TeR = W[72];
+		    TeT = W[73];
+		    iio[-WS(ios, 26)] = FMA(TeR, TeS, TeT * TeU);
+		    rio[WS(ios, 37)] = FNMS(TeT, TeS, TeR * TeU);
+	       }
+	       {
+		    E Teu, Tew, Tet, Tev;
+		    Teu = Teq - Ter;
+		    Tew = Ten - Tem;
+		    Tet = W[88];
+		    Tev = W[89];
+		    iio[-WS(ios, 18)] = FMA(Tet, Teu, Tev * Tew);
+		    rio[WS(ios, 45)] = FNMS(Tev, Teu, Tet * Tew);
+	       }
+	  }
+	  {
+	       E Tcr, Tdw, TcX, Td6, TcI, Tdt, TcS, Tdl, Tbm, TcW, TcL, TcT, Tdd, Tdx, Tdi;
+	       E Tds;
+	       {
+		    E Tcq, Td4, TbZ, Td5, TbF, TbY;
+		    Tcq = Tce + Tcp;
+		    Td4 = TcA - TcD;
+		    TbF = FNMS(KP195090322, TbE, KP980785280 * Tbx);
+		    TbY = FMA(KP195090322, TbQ, KP980785280 * TbX);
+		    TbZ = TbF + TbY;
+		    Td5 = TbY - TbF;
+		    Tcr = TbZ + Tcq;
+		    Tdw = Td4 - Td5;
+		    TcX = Tcq - TbZ;
+		    Td6 = Td4 + Td5;
+	       }
+	       {
+		    E TcE, Tdk, TcH, Tdj, TcF, TcG;
+		    TcE = TcA + TcD;
+		    Tdk = Tcp - Tce;
+		    TcF = FMA(KP980785280, TbE, KP195090322 * Tbx);
+		    TcG = FNMS(KP195090322, TbX, KP980785280 * TbQ);
+		    TcH = TcF + TcG;
+		    Tdj = TcF - TcG;
+		    TcI = TcE + TcH;
+		    Tdt = Tdk - Tdj;
+		    TcS = TcE - TcH;
+		    Tdl = Tdj + Tdk;
+	       }
+	       {
+		    E TaI, TcJ, Tbl, TcK;
+		    {
+			 E Taw, TaH, Tb9, Tbk;
+			 Taw = Tak + Tav;
+			 TaH = TaD + TaG;
+			 TaI = FNMS(KP098017140, TaH, KP995184726 * Taw);
+			 TcJ = FMA(KP995184726, TaH, KP098017140 * Taw);
+			 Tb9 = TaT + Tb8;
+			 Tbk = Tbc + Tbj;
+			 Tbl = FMA(KP098017140, Tb9, KP995184726 * Tbk);
+			 TcK = FNMS(KP098017140, Tbk, KP995184726 * Tb9);
+		    }
+		    Tbm = TaI + Tbl;
+		    TcW = TcJ - TcK;
+		    TcL = TcJ + TcK;
+		    TcT = Tbl - TaI;
+	       }
+	       {
+		    E Td9, Tdg, Tdc, Tdh;
+		    {
+			 E Td7, Td8, Tda, Tdb;
+			 Td7 = TaD - TaG;
+			 Td8 = Tav - Tak;
+			 Td9 = FNMS(KP634393284, Td8, KP773010453 * Td7);
+			 Tdg = FMA(KP634393284, Td7, KP773010453 * Td8);
+			 Tda = TaT - Tb8;
+			 Tdb = Tbj - Tbc;
+			 Tdc = FMA(KP773010453, Tda, KP634393284 * Tdb);
+			 Tdh = FNMS(KP634393284, Tda, KP773010453 * Tdb);
+		    }
+		    Tdd = Td9 + Tdc;
+		    Tdx = Tdg - Tdh;
+		    Tdi = Tdg + Tdh;
+		    Tds = Tdc - Td9;
+	       }
+	       {
+		    E Tcs, TcM, Ta5, Tct;
+		    Tcs = Tbm + Tcr;
+		    TcM = TcI + TcL;
+		    Ta5 = W[0];
+		    Tct = W[1];
+		    rio[WS(ios, 1)] = FNMS(Tct, TcM, Ta5 * Tcs);
+		    iio[-WS(ios, 62)] = FMA(Ta5, TcM, Tct * Tcs);
+	       }
+	       {
+		    E Tdu, Tdy, Tdr, Tdv;
+		    Tdu = Tds + Tdt;
+		    Tdy = Tdw + Tdx;
+		    Tdr = W[16];
+		    Tdv = W[17];
+		    rio[WS(ios, 9)] = FNMS(Tdv, Tdy, Tdr * Tdu);
+		    iio[-WS(ios, 54)] = FMA(Tdr, Tdy, Tdv * Tdu);
+	       }
+	       {
+		    E TdA, TdC, Tdz, TdB;
+		    TdA = Tdw - Tdx;
+		    TdC = Tdt - Tds;
+		    Tdz = W[80];
+		    TdB = W[81];
+		    iio[-WS(ios, 22)] = FMA(Tdz, TdA, TdB * TdC);
+		    rio[WS(ios, 41)] = FNMS(TdB, TdA, Tdz * TdC);
+	       }
+	       {
+		    E TcO, TcQ, TcN, TcP;
+		    TcO = TcI - TcL;
+		    TcQ = Tcr - Tbm;
+		    TcN = W[64];
+		    TcP = W[65];
+		    iio[-WS(ios, 30)] = FMA(TcN, TcO, TcP * TcQ);
+		    rio[WS(ios, 33)] = FNMS(TcP, TcO, TcN * TcQ);
+	       }
+	       {
+		    E TcU, TcY, TcR, TcV;
+		    TcU = TcS + TcT;
+		    TcY = TcW + TcX;
+		    TcR = W[96];
+		    TcV = W[97];
+		    iio[-WS(ios, 14)] = FMA(TcR, TcU, TcV * TcY);
+		    rio[WS(ios, 49)] = FNMS(TcV, TcU, TcR * TcY);
+	       }
+	       {
+		    E Tde, Tdm, Td3, Tdf;
+		    Tde = Td6 + Tdd;
+		    Tdm = Tdi + Tdl;
+		    Td3 = W[112];
+		    Tdf = W[113];
+		    iio[-WS(ios, 6)] = FMA(Td3, Tde, Tdf * Tdm);
+		    rio[WS(ios, 57)] = FNMS(Tdf, Tde, Td3 * Tdm);
+	       }
+	       {
+		    E Tdo, Tdq, Tdn, Tdp;
+		    Tdo = Tdl - Tdi;
+		    Tdq = Td6 - Tdd;
+		    Tdn = W[48];
+		    Tdp = W[49];
+		    rio[WS(ios, 25)] = FNMS(Tdp, Tdq, Tdn * Tdo);
+		    iio[-WS(ios, 38)] = FMA(Tdn, Tdq, Tdp * Tdo);
+	       }
+	       {
+		    E Td0, Td2, TcZ, Td1;
+		    Td0 = TcX - TcW;
+		    Td2 = TcS - TcT;
+		    TcZ = W[32];
+		    Td1 = W[33];
+		    rio[WS(ios, 17)] = FNMS(Td1, Td2, TcZ * Td0);
+		    iio[-WS(ios, 46)] = FMA(TcZ, Td2, Td1 * Td0);
+	       }
+	  }
+	  {
+	       E Tfy, Thd, TgC, TgT, Tgr, Th8, Tgz, TgY, Tgb, TgD, Tgg, Tgy, TgQ, Thc, Th1;
+	       E Th9;
+	       {
+		    E Tfi, TgS, Tfx, TgR, Tfp, Tfw;
+		    Tfi = Tfa + Tfh;
+		    TgS = Tgp - Tgm;
+		    Tfp = FNMS(KP195090322, Tfo, KP980785280 * Tfl);
+		    Tfw = FMA(KP980785280, Tfs, KP195090322 * Tfv);
+		    Tfx = Tfp + Tfw;
+		    TgR = Tfw - Tfp;
+		    Tfy = Tfi + Tfx;
+		    Thd = TgS - TgR;
+		    TgC = Tfi - Tfx;
+		    TgT = TgR + TgS;
+	       }
+	       {
+		    E Tgq, TgW, Tgj, TgX, Tgh, Tgi;
+		    Tgq = Tgm + Tgp;
+		    TgW = Tfa - Tfh;
+		    Tgh = FMA(KP195090322, Tfl, KP980785280 * Tfo);
+		    Tgi = FNMS(KP195090322, Tfs, KP980785280 * Tfv);
+		    Tgj = Tgh + Tgi;
+		    TgX = Tgh - Tgi;
+		    Tgr = Tgj + Tgq;
+		    Th8 = TgW - TgX;
+		    Tgz = Tgq - Tgj;
+		    TgY = TgW + TgX;
+	       }
+	       {
+		    E TfR, Tge, Tga, Tgf;
+		    {
+			 E TfJ, TfQ, Tg2, Tg9;
+			 TfJ = TfB + TfI;
+			 TfQ = TfM + TfP;
+			 TfR = FNMS(KP098017140, TfQ, KP995184726 * TfJ);
+			 Tge = FMA(KP098017140, TfJ, KP995184726 * TfQ);
+			 Tg2 = TfU + Tg1;
+			 Tg9 = Tg5 + Tg8;
+			 Tga = FMA(KP995184726, Tg2, KP098017140 * Tg9);
+			 Tgf = FNMS(KP098017140, Tg2, KP995184726 * Tg9);
+		    }
+		    Tgb = TfR + Tga;
+		    TgD = Tge - Tgf;
+		    Tgg = Tge + Tgf;
+		    Tgy = Tga - TfR;
+	       }
+	       {
+		    E TgM, TgZ, TgP, Th0;
+		    {
+			 E TgK, TgL, TgN, TgO;
+			 TgK = TfP - TfM;
+			 TgL = TfB - TfI;
+			 TgM = FNMS(KP634393284, TgL, KP773010453 * TgK);
+			 TgZ = FMA(KP773010453, TgL, KP634393284 * TgK);
+			 TgN = TfU - Tg1;
+			 TgO = Tg8 - Tg5;
+			 TgP = FMA(KP634393284, TgN, KP773010453 * TgO);
+			 Th0 = FNMS(KP634393284, TgO, KP773010453 * TgN);
+		    }
+		    TgQ = TgM + TgP;
+		    Thc = TgZ - Th0;
+		    Th1 = TgZ + Th0;
+		    Th9 = TgP - TgM;
+	       }
+	       {
+		    E Tgc, Tgs, Tf7, Tgd;
+		    Tgc = Tfy + Tgb;
+		    Tgs = Tgg + Tgr;
+		    Tf7 = W[124];
+		    Tgd = W[125];
+		    iio[0] = FMA(Tf7, Tgc, Tgd * Tgs);
+		    rio[WS(ios, 63)] = FNMS(Tgd, Tgc, Tf7 * Tgs);
+	       }
+	       {
+		    E Tha, The, Th7, Thb;
+		    Tha = Th8 + Th9;
+		    The = Thc + Thd;
+		    Th7 = W[108];
+		    Thb = W[109];
+		    iio[-WS(ios, 8)] = FMA(Th7, Tha, Thb * The);
+		    rio[WS(ios, 55)] = FNMS(Thb, Tha, Th7 * The);
+	       }
+	       {
+		    E Thg, Thi, Thf, Thh;
+		    Thg = Thd - Thc;
+		    Thi = Th8 - Th9;
+		    Thf = W[44];
+		    Thh = W[45];
+		    rio[WS(ios, 23)] = FNMS(Thh, Thi, Thf * Thg);
+		    iio[-WS(ios, 40)] = FMA(Thf, Thi, Thh * Thg);
+	       }
+	       {
+		    E Tgu, Tgw, Tgt, Tgv;
+		    Tgu = Tgr - Tgg;
+		    Tgw = Tfy - Tgb;
+		    Tgt = W[60];
+		    Tgv = W[61];
+		    rio[WS(ios, 31)] = FNMS(Tgv, Tgw, Tgt * Tgu);
+		    iio[-WS(ios, 32)] = FMA(Tgt, Tgw, Tgv * Tgu);
+	       }
+	       {
+		    E TgA, TgE, Tgx, TgB;
+		    TgA = Tgy + Tgz;
+		    TgE = TgC + TgD;
+		    Tgx = W[28];
+		    TgB = W[29];
+		    rio[WS(ios, 15)] = FNMS(TgB, TgE, Tgx * TgA);
+		    iio[-WS(ios, 48)] = FMA(Tgx, TgE, TgB * TgA);
+	       }
+	       {
+		    E TgU, Th2, TgJ, TgV;
+		    TgU = TgQ + TgT;
+		    Th2 = TgY + Th1;
+		    TgJ = W[12];
+		    TgV = W[13];
+		    rio[WS(ios, 7)] = FNMS(TgV, Th2, TgJ * TgU);
+		    iio[-WS(ios, 56)] = FMA(TgJ, Th2, TgV * TgU);
+	       }
+	       {
+		    E Th4, Th6, Th3, Th5;
+		    Th4 = TgY - Th1;
+		    Th6 = TgT - TgQ;
+		    Th3 = W[76];
+		    Th5 = W[77];
+		    iio[-WS(ios, 24)] = FMA(Th3, Th4, Th5 * Th6);
+		    rio[WS(ios, 39)] = FNMS(Th5, Th4, Th3 * Th6);
+	       }
+	       {
+		    E TgG, TgI, TgF, TgH;
+		    TgG = TgC - TgD;
+		    TgI = Tgz - Tgy;
+		    TgF = W[92];
+		    TgH = W[93];
+		    iio[-WS(ios, 16)] = FMA(TgF, TgG, TgH * TgI);
+		    rio[WS(ios, 47)] = FNMS(TgH, TgG, TgF * TgI);
+	       }
+	  }
+	  {
+	       E ThJ, TiG, Ti7, Tig, ThS, TiD, Ti2, Tiv, Thy, Ti6, ThV, Ti3, Tin, TiH, Tis;
+	       E TiC;
+	       {
+		    E ThI, Tie, ThF, Tif, ThB, ThE;
+		    ThI = ThG + ThH;
+		    Tie = ThM - ThN;
+		    ThB = FNMS(KP555570233, ThA, KP831469612 * Thz);
+		    ThE = FNMS(KP555570233, ThD, KP831469612 * ThC);
+		    ThF = ThB + ThE;
+		    Tif = ThE - ThB;
+		    ThJ = ThF + ThI;
+		    TiG = Tie - Tif;
+		    Ti7 = ThI - ThF;
+		    Tig = Tie + Tif;
+	       }
+	       {
+		    E ThO, Tiu, ThR, Tit, ThP, ThQ;
+		    ThO = ThM + ThN;
+		    Tiu = ThH - ThG;
+		    ThP = FMA(KP831469612, ThA, KP555570233 * Thz);
+		    ThQ = FMA(KP831469612, ThD, KP555570233 * ThC);
+		    ThR = ThP - ThQ;
+		    Tit = ThP + ThQ;
+		    ThS = ThO + ThR;
+		    TiD = Tiu - Tit;
+		    Ti2 = ThO - ThR;
+		    Tiv = Tit + Tiu;
+	       }
+	       {
+		    E Thq, ThT, Thx, ThU;
+		    {
+			 E Thm, Thp, Tht, Thw;
+			 Thm = Thk + Thl;
+			 Thp = Thn + Tho;
+			 Thq = FNMS(KP290284677, Thp, KP956940335 * Thm);
+			 ThT = FMA(KP956940335, Thp, KP290284677 * Thm);
+			 Tht = Thr - Ths;
+			 Thw = Thu + Thv;
+			 Thx = FMA(KP290284677, Tht, KP956940335 * Thw);
+			 ThU = FNMS(KP290284677, Thw, KP956940335 * Tht);
+		    }
+		    Thy = Thq + Thx;
+		    Ti6 = ThT - ThU;
+		    ThV = ThT + ThU;
+		    Ti3 = Thx - Thq;
+	       }
+	       {
+		    E Tij, Tiq, Tim, Tir;
+		    {
+			 E Tih, Tii, Tik, Til;
+			 Tih = Thn - Tho;
+			 Tii = Thl - Thk;
+			 Tij = FNMS(KP471396736, Tii, KP881921264 * Tih);
+			 Tiq = FMA(KP471396736, Tih, KP881921264 * Tii);
+			 Tik = Thv - Thu;
+			 Til = Ths + Thr;
+			 Tim = FNMS(KP881921264, Til, KP471396736 * Tik);
+			 Tir = FMA(KP471396736, Til, KP881921264 * Tik);
+		    }
+		    Tin = Tij + Tim;
+		    TiH = Tiq - Tir;
+		    Tis = Tiq + Tir;
+		    TiC = Tim - Tij;
+	       }
+	       {
+		    E ThK, ThW, Thj, ThL;
+		    ThK = Thy + ThJ;
+		    ThW = ThS + ThV;
+		    Thj = W[4];
+		    ThL = W[5];
+		    rio[WS(ios, 3)] = FNMS(ThL, ThW, Thj * ThK);
+		    iio[-WS(ios, 60)] = FMA(Thj, ThW, ThL * ThK);
+	       }
+	       {
+		    E TiE, TiI, TiB, TiF;
+		    TiE = TiC + TiD;
+		    TiI = TiG + TiH;
+		    TiB = W[20];
+		    TiF = W[21];
+		    rio[WS(ios, 11)] = FNMS(TiF, TiI, TiB * TiE);
+		    iio[-WS(ios, 52)] = FMA(TiB, TiI, TiF * TiE);
+	       }
+	       {
+		    E TiK, TiM, TiJ, TiL;
+		    TiK = TiG - TiH;
+		    TiM = TiD - TiC;
+		    TiJ = W[84];
+		    TiL = W[85];
+		    iio[-WS(ios, 20)] = FMA(TiJ, TiK, TiL * TiM);
+		    rio[WS(ios, 43)] = FNMS(TiL, TiK, TiJ * TiM);
+	       }
+	       {
+		    E ThY, Ti0, ThX, ThZ;
+		    ThY = ThS - ThV;
+		    Ti0 = ThJ - Thy;
+		    ThX = W[68];
+		    ThZ = W[69];
+		    iio[-WS(ios, 28)] = FMA(ThX, ThY, ThZ * Ti0);
+		    rio[WS(ios, 35)] = FNMS(ThZ, ThY, ThX * Ti0);
+	       }
+	       {
+		    E Ti4, Ti8, Ti1, Ti5;
+		    Ti4 = Ti2 + Ti3;
+		    Ti8 = Ti6 + Ti7;
+		    Ti1 = W[100];
+		    Ti5 = W[101];
+		    iio[-WS(ios, 12)] = FMA(Ti1, Ti4, Ti5 * Ti8);
+		    rio[WS(ios, 51)] = FNMS(Ti5, Ti4, Ti1 * Ti8);
+	       }
+	       {
+		    E Tio, Tiw, Tid, Tip;
+		    Tio = Tig + Tin;
+		    Tiw = Tis + Tiv;
+		    Tid = W[116];
+		    Tip = W[117];
+		    iio[-WS(ios, 4)] = FMA(Tid, Tio, Tip * Tiw);
+		    rio[WS(ios, 59)] = FNMS(Tip, Tio, Tid * Tiw);
+	       }
+	       {
+		    E Tiy, TiA, Tix, Tiz;
+		    Tiy = Tiv - Tis;
+		    TiA = Tig - Tin;
+		    Tix = W[52];
+		    Tiz = W[53];
+		    rio[WS(ios, 27)] = FNMS(Tiz, TiA, Tix * Tiy);
+		    iio[-WS(ios, 36)] = FMA(Tix, TiA, Tiz * Tiy);
+	       }
+	       {
+		    E Tia, Tic, Ti9, Tib;
+		    Tia = Ti7 - Ti6;
+		    Tic = Ti2 - Ti3;
+		    Ti9 = W[36];
+		    Tib = W[37];
+		    rio[WS(ios, 19)] = FNMS(Tib, Tic, Ti9 * Tia);
+		    iio[-WS(ios, 44)] = FMA(Ti9, Tic, Tib * Tia);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 64},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 64, "hb_64", twinstr, {808, 270, 230, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_64) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_64, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_7.c b/src/fftw3/rdft/codelets/hc2r/hb_7.c
new file mode 100644
index 0000000..4993e5e
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_7.c
@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:26 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 7 -dif -name hb_7 -include hb.h */
+
+/*
+ * This function contains 72 FP additions, 60 FP multiplications,
+ * (or, 36 additions, 24 multiplications, 36 fused multiply/add),
+ * 36 stack variables, and 28 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_7.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_7.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_7.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_7(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP222520933, +0.222520933956314404288902564496794759466355569);
+     DK(KP900968867, +0.900968867902419126236102319507445051165919162);
+     DK(KP623489801, +0.623489801858733530525004884004239810632274731);
+     DK(KP781831482, +0.781831482468029808708444526674057750232334519);
+     DK(KP974927912, +0.974927912181823607018131682993931217232785801);
+     DK(KP433883739, +0.433883739117558120475768332848358754609990728);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 12) {
+	  E T1, Ta, T4, T7, Tq, TI, TR, TU, TE, Tt, Tb, Tk, Te, Th, Tx;
+	  E TF, TV, TQ, TJ, Tm;
+	  {
+	       E To, Tn, Tp, T8, T9;
+	       T1 = rio[0];
+	       T8 = rio[WS(ios, 3)];
+	       T9 = iio[-WS(ios, 4)];
+	       Ta = T8 + T9;
+	       To = T8 - T9;
+	       {
+		    E T2, T3, T5, T6;
+		    T2 = rio[WS(ios, 1)];
+		    T3 = iio[-WS(ios, 6)];
+		    T4 = T2 + T3;
+		    Tn = T2 - T3;
+		    T5 = rio[WS(ios, 2)];
+		    T6 = iio[-WS(ios, 5)];
+		    T7 = T5 + T6;
+		    Tp = T5 - T6;
+	       }
+	       Tq = FMA(KP433883739, Tn, KP974927912 * To) - (KP781831482 * Tp);
+	       TI = FMA(KP781831482, Tn, KP974927912 * Tp) + (KP433883739 * To);
+	       TR = FNMS(KP781831482, To, KP974927912 * Tn) - (KP433883739 * Tp);
+	       TU = FMA(KP623489801, Ta, T1) + FNMA(KP900968867, T7, KP222520933 * T4);
+	       TE = FMA(KP623489801, T4, T1) + FNMA(KP900968867, Ta, KP222520933 * T7);
+	       Tt = FMA(KP623489801, T7, T1) + FNMA(KP222520933, Ta, KP900968867 * T4);
+	  }
+	  {
+	       E Tv, Tw, Tu, Ti, Tj;
+	       Tb = iio[0];
+	       Ti = rio[WS(ios, 4)];
+	       Tj = iio[-WS(ios, 3)];
+	       Tk = Ti - Tj;
+	       Tv = Ti + Tj;
+	       {
+		    E Tc, Td, Tf, Tg;
+		    Tc = iio[-WS(ios, 1)];
+		    Td = rio[WS(ios, 6)];
+		    Te = Tc - Td;
+		    Tw = Tc + Td;
+		    Tf = rio[WS(ios, 5)];
+		    Tg = iio[-WS(ios, 2)];
+		    Th = Tf - Tg;
+		    Tu = Tf + Tg;
+	       }
+	       Tx = FNMS(KP974927912, Tv, KP781831482 * Tu) - (KP433883739 * Tw);
+	       TF = FMA(KP781831482, Tw, KP974927912 * Tu) + (KP433883739 * Tv);
+	       TV = FMA(KP433883739, Tu, KP781831482 * Tv) - (KP974927912 * Tw);
+	       TQ = FMA(KP900968867, Th, Tb) + FNMA(KP623489801, Tk, KP222520933 * Te);
+	       TJ = FMA(KP623489801, Te, Tb) + FMA(KP222520933, Th, KP900968867 * Tk);
+	       Tm = FMA(KP222520933, Tk, Tb) + FNMA(KP623489801, Th, KP900968867 * Te);
+	  }
+	  rio[0] = T1 + T4 + T7 + Ta;
+	  iio[-WS(ios, 6)] = Tb + Te - (Th + Tk);
+	  {
+	       E TM, TO, TL, TN;
+	       TM = TJ - TI;
+	       TO = TE + TF;
+	       TL = W[10];
+	       TN = W[11];
+	       iio[0] = FMA(TL, TM, TN * TO);
+	       rio[WS(ios, 6)] = FNMS(TN, TM, TL * TO);
+	  }
+	  {
+	       E TY, T10, TX, TZ;
+	       TY = TU + TV;
+	       T10 = TR + TQ;
+	       TX = W[2];
+	       TZ = W[3];
+	       rio[WS(ios, 2)] = FNMS(TZ, T10, TX * TY);
+	       iio[-WS(ios, 4)] = FMA(TX, T10, TZ * TY);
+	  }
+	  {
+	       E Tr, Ty, Tl, Ts;
+	       Tr = Tm - Tq;
+	       Ty = Tt - Tx;
+	       Tl = W[6];
+	       Ts = W[7];
+	       iio[-WS(ios, 2)] = FMA(Tl, Tr, Ts * Ty);
+	       rio[WS(ios, 4)] = FNMS(Ts, Tr, Tl * Ty);
+	  }
+	  {
+	       E TA, TC, Tz, TB;
+	       TA = Tt + Tx;
+	       TC = Tq + Tm;
+	       Tz = W[4];
+	       TB = W[5];
+	       rio[WS(ios, 3)] = FNMS(TB, TC, Tz * TA);
+	       iio[-WS(ios, 3)] = FMA(Tz, TC, TB * TA);
+	  }
+	  {
+	       E TS, TW, TP, TT;
+	       TS = TQ - TR;
+	       TW = TU - TV;
+	       TP = W[8];
+	       TT = W[9];
+	       iio[-WS(ios, 1)] = FMA(TP, TS, TT * TW);
+	       rio[WS(ios, 5)] = FNMS(TT, TS, TP * TW);
+	  }
+	  {
+	       E TG, TK, TD, TH;
+	       TG = TE - TF;
+	       TK = TI + TJ;
+	       TD = W[0];
+	       TH = W[1];
+	       rio[WS(ios, 1)] = FNMS(TH, TK, TD * TG);
+	       iio[-WS(ios, 5)] = FMA(TD, TK, TH * TG);
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 7},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 7, "hb_7", twinstr, {36, 24, 36, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_7) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_7, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_8.c b/src/fftw3/rdft/codelets/hc2r/hb_8.c
new file mode 100644
index 0000000..58cc25e
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_8.c
@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:28 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 8 -dif -name hb_8 -include hb.h */
+
+/*
+ * This function contains 66 FP additions, 32 FP multiplications,
+ * (or, 52 additions, 18 multiplications, 14 fused multiply/add),
+ * 30 stack variables, and 32 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_8.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_8.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_8.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_8(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 14) {
+	  E T7, T18, T1d, Tg, Tx, TT, TY, TG, Te, TZ, T10, Tn, Tq, TM, TP;
+	  E TH;
+	  {
+	       E T3, TR, Tt, TX, T6, TW, Tw, TS;
+	       {
+		    E T1, T2, Tr, Ts;
+		    T1 = rio[0];
+		    T2 = iio[-WS(ios, 4)];
+		    T3 = T1 + T2;
+		    TR = T1 - T2;
+		    Tr = iio[0];
+		    Ts = rio[WS(ios, 4)];
+		    Tt = Tr - Ts;
+		    TX = Tr + Ts;
+	       }
+	       {
+		    E T4, T5, Tu, Tv;
+		    T4 = rio[WS(ios, 2)];
+		    T5 = iio[-WS(ios, 6)];
+		    T6 = T4 + T5;
+		    TW = T4 - T5;
+		    Tu = iio[-WS(ios, 2)];
+		    Tv = rio[WS(ios, 6)];
+		    Tw = Tu - Tv;
+		    TS = Tu + Tv;
+	       }
+	       T7 = T3 + T6;
+	       T18 = TX - TW;
+	       T1d = TR + TS;
+	       Tg = T3 - T6;
+	       Tx = Tt - Tw;
+	       TT = TR - TS;
+	       TY = TW + TX;
+	       TG = Tt + Tw;
+	  }
+	  {
+	       E Ta, TO, Tj, TN, Td, TK, Tm, TL;
+	       {
+		    E T8, T9, Th, Ti;
+		    T8 = rio[WS(ios, 1)];
+		    T9 = iio[-WS(ios, 5)];
+		    Ta = T8 + T9;
+		    TO = T8 - T9;
+		    Th = iio[-WS(ios, 1)];
+		    Ti = rio[WS(ios, 5)];
+		    Tj = Th - Ti;
+		    TN = Th + Ti;
+	       }
+	       {
+		    E Tb, Tc, Tk, Tl;
+		    Tb = iio[-WS(ios, 7)];
+		    Tc = rio[WS(ios, 3)];
+		    Td = Tb + Tc;
+		    TK = Tb - Tc;
+		    Tk = iio[-WS(ios, 3)];
+		    Tl = rio[WS(ios, 7)];
+		    Tm = Tk - Tl;
+		    TL = Tl + Tk;
+	       }
+	       Te = Ta + Td;
+	       TZ = TO + TN;
+	       T10 = TK + TL;
+	       Tn = Tj - Tm;
+	       Tq = Td - Ta;
+	       TM = TK - TL;
+	       TP = TN - TO;
+	       TH = Tj + Tm;
+	  }
+	  rio[0] = T7 + Te;
+	  iio[-WS(ios, 7)] = TH + TG;
+	  {
+	       E To, Ty, Tf, Tp;
+	       To = Tg + Tn;
+	       Ty = Tq + Tx;
+	       Tf = W[10];
+	       Tp = W[11];
+	       rio[WS(ios, 6)] = FNMS(Tp, Ty, Tf * To);
+	       iio[-WS(ios, 1)] = FMA(Tp, To, Tf * Ty);
+	  }
+	  {
+	       E TE, TI, TD, TF;
+	       TE = T7 - Te;
+	       TI = TG - TH;
+	       TD = W[6];
+	       TF = W[7];
+	       rio[WS(ios, 4)] = FNMS(TF, TI, TD * TE);
+	       iio[-WS(ios, 3)] = FMA(TF, TE, TD * TI);
+	  }
+	  {
+	       E TA, TC, Tz, TB;
+	       TA = Tg - Tn;
+	       TC = Tx - Tq;
+	       Tz = W[2];
+	       TB = W[3];
+	       rio[WS(ios, 2)] = FNMS(TB, TC, Tz * TA);
+	       iio[-WS(ios, 5)] = FMA(TB, TA, Tz * TC);
+	  }
+	  {
+	       E TU, T16, T12, T14, TQ, T11;
+	       TQ = KP707106781 * (TM - TP);
+	       TU = TQ + TT;
+	       T16 = TT - TQ;
+	       T11 = KP707106781 * (TZ - T10);
+	       T12 = TY + T11;
+	       T14 = TY - T11;
+	       {
+		    E TJ, TV, T13, T15;
+		    TJ = W[0];
+		    TV = W[1];
+		    rio[WS(ios, 1)] = FNMS(TV, T12, TJ * TU);
+		    iio[-WS(ios, 6)] = FMA(TJ, T12, TV * TU);
+		    T13 = W[8];
+		    T15 = W[9];
+		    iio[-WS(ios, 2)] = FMA(T13, T14, T15 * T16);
+		    rio[WS(ios, 5)] = FNMS(T15, T14, T13 * T16);
+	       }
+	  }
+	  {
+	       E T1a, T1i, T1e, T1g, T19, T1c;
+	       T19 = KP707106781 * (TP + TM);
+	       T1a = T18 + T19;
+	       T1i = T18 - T19;
+	       T1c = KP707106781 * (TZ + T10);
+	       T1e = T1c + T1d;
+	       T1g = T1d - T1c;
+	       {
+		    E T17, T1b, T1f, T1h;
+		    T17 = W[12];
+		    T1b = W[13];
+		    iio[0] = FMA(T17, T1a, T1b * T1e);
+		    rio[WS(ios, 7)] = FNMS(T1b, T1a, T17 * T1e);
+		    T1f = W[4];
+		    T1h = W[5];
+		    rio[WS(ios, 3)] = FNMS(T1h, T1i, T1f * T1g);
+		    iio[-WS(ios, 4)] = FMA(T1f, T1i, T1h * T1g);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 8},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 8, "hb_8", twinstr, {52, 18, 14, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_8) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_8, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hb_9.c b/src/fftw3/rdft/codelets/hc2r/hb_9.c
new file mode 100644
index 0000000..ed3eb98
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hb_9.c
@@ -0,0 +1,248 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:31 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -sign 1 -n 9 -dif -name hb_9 -include hb.h */
+
+/*
+ * This function contains 96 FP additions, 72 FP multiplications,
+ * (or, 60 additions, 36 multiplications, 36 fused multiply/add),
+ * 53 stack variables, and 36 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hb_9.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_9.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hb_9.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hb.h"
+
+static const R *hb_9(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP642787609, +0.642787609686539326322643409907263432907559884);
+     DK(KP766044443, +0.766044443118978035202392650555416673935832457);
+     DK(KP984807753, +0.984807753012208059366743024589523013670643252);
+     DK(KP173648177, +0.173648177666930348851716626769314796000375677);
+     DK(KP342020143, +0.342020143325668733044099614682259580763083368);
+     DK(KP939692620, +0.939692620785908384054109277324731469936208134);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 16) {
+	  E T5, T1z, Tm, T18, TQ, T1i, Ta, Tf, Tg, T1A, T1B, T1C, Tx, TS, T1e;
+	  E T1k, T1b, T1j, TI, TR;
+	  {
+	       E T1, TM, T4, TP, Tl, TN, Ti, TO;
+	       T1 = rio[0];
+	       TM = iio[0];
+	       {
+		    E T2, T3, Tj, Tk;
+		    T2 = rio[WS(ios, 3)];
+		    T3 = iio[-WS(ios, 6)];
+		    T4 = T2 + T3;
+		    TP = KP866025403 * (T2 - T3);
+		    Tj = rio[WS(ios, 6)];
+		    Tk = iio[-WS(ios, 3)];
+		    Tl = KP866025403 * (Tj + Tk);
+		    TN = Tj - Tk;
+	       }
+	       T5 = T1 + T4;
+	       T1z = TM - TN;
+	       Ti = FNMS(KP500000000, T4, T1);
+	       Tm = Ti + Tl;
+	       T18 = Ti - Tl;
+	       TO = FMA(KP500000000, TN, TM);
+	       TQ = TO - TP;
+	       T1i = TP + TO;
+	  }
+	  {
+	       E T6, T9, Ty, TG, TD, TE, TB, TF, Tb, Te, Tn, Tv, Tt, Ts, Tq;
+	       E Tu;
+	       {
+		    E T7, T8, Tz, TA;
+		    T6 = rio[WS(ios, 1)];
+		    T7 = rio[WS(ios, 4)];
+		    T8 = iio[-WS(ios, 7)];
+		    T9 = T7 + T8;
+		    Ty = FNMS(KP500000000, T9, T6);
+		    TG = KP866025403 * (T7 - T8);
+		    TD = iio[-WS(ios, 1)];
+		    Tz = rio[WS(ios, 7)];
+		    TA = iio[-WS(ios, 4)];
+		    TE = Tz - TA;
+		    TB = KP866025403 * (Tz + TA);
+		    TF = FMA(KP500000000, TE, TD);
+	       }
+	       {
+		    E Tc, Td, To, Tp;
+		    Tb = rio[WS(ios, 2)];
+		    Tc = iio[-WS(ios, 5)];
+		    Td = iio[-WS(ios, 8)];
+		    Te = Tc + Td;
+		    Tn = FNMS(KP500000000, Te, Tb);
+		    Tv = KP866025403 * (Tc - Td);
+		    Tt = iio[-WS(ios, 2)];
+		    To = rio[WS(ios, 5)];
+		    Tp = rio[WS(ios, 8)];
+		    Ts = To + Tp;
+		    Tq = KP866025403 * (To - Tp);
+		    Tu = FMA(KP500000000, Ts, Tt);
+	       }
+	       {
+		    E Tr, Tw, T1c, T1d;
+		    Ta = T6 + T9;
+		    Tf = Tb + Te;
+		    Tg = Ta + Tf;
+		    T1A = TD - TE;
+		    T1B = Tt - Ts;
+		    T1C = T1A + T1B;
+		    Tr = Tn - Tq;
+		    Tw = Tu - Tv;
+		    Tx = FMA(KP939692620, Tr, KP342020143 * Tw);
+		    TS = FNMS(KP939692620, Tw, KP342020143 * Tr);
+		    T1c = Tn + Tq;
+		    T1d = Tv + Tu;
+		    T1e = FNMS(KP984807753, T1d, KP173648177 * T1c);
+		    T1k = FMA(KP984807753, T1c, KP173648177 * T1d);
+		    {
+			 E T19, T1a, TC, TH;
+			 T19 = Ty - TB;
+			 T1a = TG + TF;
+			 T1b = FNMS(KP642787609, T1a, KP766044443 * T19);
+			 T1j = FMA(KP766044443, T1a, KP642787609 * T19);
+			 TC = Ty + TB;
+			 TH = TF - TG;
+			 TI = FNMS(KP984807753, TH, KP173648177 * TC);
+			 TR = FMA(KP173648177, TH, KP984807753 * TC);
+		    }
+	       }
+	  }
+	  rio[0] = T5 + Tg;
+	  {
+	       E TX, T11, TK, T10, TU, TW, TJ, TT, Th, TL;
+	       TX = KP866025403 * (TI + Tx);
+	       T11 = KP866025403 * (TS - TR);
+	       TJ = Tx - TI;
+	       TK = Tm - TJ;
+	       T10 = FMA(KP500000000, TJ, Tm);
+	       TT = TR + TS;
+	       TU = TQ + TT;
+	       TW = FNMS(KP500000000, TT, TQ);
+	       Th = W[2];
+	       TL = W[3];
+	       rio[WS(ios, 2)] = FNMS(TL, TU, Th * TK);
+	       iio[-WS(ios, 6)] = FMA(Th, TU, TL * TK);
+	       {
+		    E T14, T16, T13, T15;
+		    T14 = TW + TX;
+		    T16 = T11 + T10;
+		    T13 = W[8];
+		    T15 = W[9];
+		    iio[-WS(ios, 3)] = FMA(T13, T14, T15 * T16);
+		    rio[WS(ios, 5)] = FNMS(T15, T14, T13 * T16);
+	       }
+	       {
+		    E TY, T12, TV, TZ;
+		    TY = TW - TX;
+		    T12 = T10 - T11;
+		    TV = W[14];
+		    TZ = W[15];
+		    iio[0] = FMA(TV, TY, TZ * T12);
+		    rio[WS(ios, 8)] = FNMS(TZ, TY, TV * T12);
+	       }
+	  }
+	  iio[-WS(ios, 8)] = T1z + T1C;
+	  {
+	       E T1G, T1O, T1K, T1M;
+	       {
+		    E T1E, T1F, T1I, T1J;
+		    T1E = FNMS(KP500000000, T1C, T1z);
+		    T1F = KP866025403 * (Ta - Tf);
+		    T1G = T1E - T1F;
+		    T1O = T1F + T1E;
+		    T1I = FNMS(KP500000000, Tg, T5);
+		    T1J = KP866025403 * (T1B - T1A);
+		    T1K = T1I - T1J;
+		    T1M = T1I + T1J;
+	       }
+	       {
+		    E T1D, T1H, T1L, T1N;
+		    T1D = W[10];
+		    T1H = W[11];
+		    iio[-WS(ios, 2)] = FMA(T1D, T1G, T1H * T1K);
+		    rio[WS(ios, 6)] = FNMS(T1H, T1G, T1D * T1K);
+		    T1L = W[4];
+		    T1N = W[5];
+		    rio[WS(ios, 3)] = FNMS(T1N, T1O, T1L * T1M);
+		    iio[-WS(ios, 5)] = FMA(T1L, T1O, T1N * T1M);
+	       }
+	  }
+	  {
+	       E T1p, T1t, T1g, T1s, T1m, T1o, T1f, T1l, T17, T1h;
+	       T1p = KP866025403 * (T1b - T1e);
+	       T1t = KP866025403 * (T1k - T1j);
+	       T1f = T1b + T1e;
+	       T1g = T18 + T1f;
+	       T1s = FNMS(KP500000000, T1f, T18);
+	       T1l = T1j + T1k;
+	       T1m = T1i + T1l;
+	       T1o = FNMS(KP500000000, T1l, T1i);
+	       T17 = W[0];
+	       T1h = W[1];
+	       rio[WS(ios, 1)] = FNMS(T1h, T1m, T17 * T1g);
+	       iio[-WS(ios, 7)] = FMA(T1h, T1g, T17 * T1m);
+	       {
+		    E T1q, T1u, T1n, T1r;
+		    T1q = T1o - T1p;
+		    T1u = T1s - T1t;
+		    T1n = W[12];
+		    T1r = W[13];
+		    iio[-WS(ios, 1)] = FMA(T1n, T1q, T1r * T1u);
+		    rio[WS(ios, 7)] = FNMS(T1r, T1q, T1n * T1u);
+	       }
+	       {
+		    E T1w, T1y, T1v, T1x;
+		    T1w = T1s + T1t;
+		    T1y = T1p + T1o;
+		    T1v = W[6];
+		    T1x = W[7];
+		    rio[WS(ios, 4)] = FNMS(T1x, T1y, T1v * T1w);
+		    iio[-WS(ios, 4)] = FMA(T1v, T1y, T1x * T1w);
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 9},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 9, "hb_9", twinstr, {60, 36, 36, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hb_9) (planner *p) {
+     X(khc2hc_dif_register) (p, hb_9, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_10.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_10.c
new file mode 100644
index 0000000..df8c872
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_10.c
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:12:05 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 10 -name hc2rIII_10 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 32 FP additions, 16 FP multiplications,
+ * (or, 26 additions, 10 multiplications, 6 fused multiply/add),
+ * 22 stack variables, and 20 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_10.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_10.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_10.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_10(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP1_902113032, +1.902113032590307144232878666758764286811397268);
+     DK(KP1_175570504, +1.175570504584946258337411909278145537195304875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_118033988, +1.118033988749894848204586834365638117720309180);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T1, To, T8, Tq, Ta, Tp, Te, Ts, Th, Tn;
+	  T1 = ri[WS(ris, 2)];
+	  To = ii[WS(iis, 2)];
+	  {
+	       E T2, T3, T4, T5, T6, T7;
+	       T2 = ri[WS(ris, 4)];
+	       T3 = ri[0];
+	       T4 = T2 + T3;
+	       T5 = ri[WS(ris, 3)];
+	       T6 = ri[WS(ris, 1)];
+	       T7 = T5 + T6;
+	       T8 = T4 + T7;
+	       Tq = T5 - T6;
+	       Ta = KP1_118033988 * (T7 - T4);
+	       Tp = T2 - T3;
+	  }
+	  {
+	       E Tc, Td, Tm, Tf, Tg, Tl;
+	       Tc = ii[WS(iis, 4)];
+	       Td = ii[0];
+	       Tm = Tc + Td;
+	       Tf = ii[WS(iis, 1)];
+	       Tg = ii[WS(iis, 3)];
+	       Tl = Tg + Tf;
+	       Te = Tc - Td;
+	       Ts = KP1_118033988 * (Tl + Tm);
+	       Th = Tf - Tg;
+	       Tn = Tl - Tm;
+	  }
+	  O[0] = KP2_000000000 * (T1 + T8);
+	  O[WS(os, 5)] = KP2_000000000 * (Tn - To);
+	  {
+	       E Ti, Tj, Tb, Tk, T9;
+	       Ti = FNMS(KP1_902113032, Th, KP1_175570504 * Te);
+	       Tj = FMA(KP1_175570504, Th, KP1_902113032 * Te);
+	       T9 = FNMS(KP2_000000000, T1, KP500000000 * T8);
+	       Tb = T9 - Ta;
+	       Tk = T9 + Ta;
+	       O[WS(os, 2)] = Tb + Ti;
+	       O[WS(os, 6)] = Tk + Tj;
+	       O[WS(os, 8)] = Ti - Tb;
+	       O[WS(os, 4)] = Tj - Tk;
+	  }
+	  {
+	       E Tr, Tv, Tu, Tw, Tt;
+	       Tr = FMA(KP1_902113032, Tp, KP1_175570504 * Tq);
+	       Tv = FNMS(KP1_175570504, Tp, KP1_902113032 * Tq);
+	       Tt = FMA(KP500000000, Tn, KP2_000000000 * To);
+	       Tu = Ts + Tt;
+	       Tw = Tt - Ts;
+	       O[WS(os, 1)] = -(Tr + Tu);
+	       O[WS(os, 7)] = Tw - Tv;
+	       O[WS(os, 9)] = Tr - Tu;
+	       O[WS(os, 3)] = Tv + Tw;
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 10, "hc2rIII_10", {26, 10, 6, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_10) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_10, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_12.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_12.c
new file mode 100644
index 0000000..0a36bf0
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_12.c
@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:12:07 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 12 -name hc2rIII_12 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 42 FP additions, 20 FP multiplications,
+ * (or, 38 additions, 16 multiplications, 4 fused multiply/add),
+ * 25 stack variables, and 24 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_12.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_12.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_12.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_12(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T5, Tw, Tb, Te, Tx, Ts, Ta, TA, Tg, Tj, Tz, Tp, Tt, Tu;
+	  {
+	       E T1, T2, T3, T4;
+	       T1 = ri[WS(ris, 1)];
+	       T2 = ri[WS(ris, 5)];
+	       T3 = ri[WS(ris, 2)];
+	       T4 = T2 + T3;
+	       T5 = T1 + T4;
+	       Tw = KP866025403 * (T2 - T3);
+	       Tb = FNMS(KP500000000, T4, T1);
+	  }
+	  {
+	       E Tq, Tc, Td, Tr;
+	       Tq = ii[WS(iis, 1)];
+	       Tc = ii[WS(iis, 5)];
+	       Td = ii[WS(iis, 2)];
+	       Tr = Td - Tc;
+	       Te = KP866025403 * (Tc + Td);
+	       Tx = FMA(KP500000000, Tr, Tq);
+	       Ts = Tq - Tr;
+	  }
+	  {
+	       E T6, T7, T8, T9;
+	       T6 = ri[WS(ris, 4)];
+	       T7 = ri[0];
+	       T8 = ri[WS(ris, 3)];
+	       T9 = T7 + T8;
+	       Ta = T6 + T9;
+	       TA = KP866025403 * (T7 - T8);
+	       Tg = FNMS(KP500000000, T9, T6);
+	  }
+	  {
+	       E To, Th, Ti, Tn;
+	       To = ii[WS(iis, 4)];
+	       Th = ii[0];
+	       Ti = ii[WS(iis, 3)];
+	       Tn = Ti - Th;
+	       Tj = KP866025403 * (Th + Ti);
+	       Tz = FMA(KP500000000, Tn, To);
+	       Tp = Tn - To;
+	  }
+	  O[0] = KP2_000000000 * (T5 + Ta);
+	  O[WS(os, 6)] = KP2_000000000 * (Ts + Tp);
+	  Tt = Tp - Ts;
+	  Tu = T5 - Ta;
+	  O[WS(os, 3)] = KP1_414213562 * (Tt - Tu);
+	  O[WS(os, 9)] = KP1_414213562 * (Tu + Tt);
+	  {
+	       E Tf, Tk, Tv, Ty, TB, TC;
+	       Tf = Tb - Te;
+	       Tk = Tg + Tj;
+	       Tv = Tf - Tk;
+	       Ty = Tw + Tx;
+	       TB = Tz - TA;
+	       TC = Ty + TB;
+	       O[WS(os, 4)] = -(KP2_000000000 * (Tf + Tk));
+	       O[WS(os, 10)] = KP2_000000000 * (TB - Ty);
+	       O[WS(os, 1)] = KP1_414213562 * (Tv - TC);
+	       O[WS(os, 7)] = KP1_414213562 * (Tv + TC);
+	  }
+	  {
+	       E Tl, Tm, TF, TD, TE, TG;
+	       Tl = Tb + Te;
+	       Tm = Tg - Tj;
+	       TF = Tm - Tl;
+	       TD = TA + Tz;
+	       TE = Tx - Tw;
+	       TG = TE + TD;
+	       O[WS(os, 8)] = KP2_000000000 * (Tl + Tm);
+	       O[WS(os, 5)] = KP1_414213562 * (TF + TG);
+	       O[WS(os, 2)] = KP2_000000000 * (TD - TE);
+	       O[WS(os, 11)] = KP1_414213562 * (TF - TG);
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 12, "hc2rIII_12", {38, 16, 4, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_12) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_12, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_15.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_15.c
new file mode 100644
index 0000000..9a94a9a
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_15.c
@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:12:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 15 -name hc2rIII_15 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 64 FP additions, 26 FP multiplications,
+ * (or, 49 additions, 11 multiplications, 15 fused multiply/add),
+ * 47 stack variables, and 30 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_15.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_15.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_15.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_15(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     DK(KP433012701, +0.433012701892219323381861585376468091735701313);
+     DK(KP968245836, +0.968245836551854221294816349945599902708230426);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP1_647278207, +1.647278207092663851754840078556380006059321028);
+     DK(KP1_018073920, +1.018073920910254366901961726787815297021466329);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_118033988, +1.118033988749894848204586834365638117720309180);
+     DK(KP1_175570504, +1.175570504584946258337411909278145537195304875);
+     DK(KP1_902113032, +1.902113032590307144232878666758764286811397268);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E Tv, TD, T5, Ts, TC, T6, Tf, TW, TK, Td, Tg, TP, To, TN, TA;
+	  E TO, TQ, Tt, Tu, T12, Te, T11;
+	  Tt = ii[WS(iis, 4)];
+	  Tu = ii[WS(iis, 1)];
+	  Tv = FMA(KP1_902113032, Tt, KP1_175570504 * Tu);
+	  TD = FNMS(KP1_175570504, Tt, KP1_902113032 * Tu);
+	  {
+	       E T1, T4, Tq, T2, T3, Tr;
+	       T1 = ri[WS(ris, 7)];
+	       T2 = ri[WS(ris, 4)];
+	       T3 = ri[WS(ris, 1)];
+	       T4 = T2 + T3;
+	       Tq = KP1_118033988 * (T2 - T3);
+	       T5 = FMA(KP2_000000000, T4, T1);
+	       Tr = FNMS(KP500000000, T4, T1);
+	       Ts = Tq + Tr;
+	       TC = Tr - Tq;
+	  }
+	  {
+	       E Tc, TJ, T9, TI;
+	       T6 = ri[WS(ris, 2)];
+	       {
+		    E Ta, Tb, T7, T8;
+		    Ta = ri[WS(ris, 3)];
+		    Tb = ri[WS(ris, 6)];
+		    Tc = Ta + Tb;
+		    TJ = Ta - Tb;
+		    T7 = ri[0];
+		    T8 = ri[WS(ris, 5)];
+		    T9 = T7 + T8;
+		    TI = T7 - T8;
+	       }
+	       Tf = KP559016994 * (T9 - Tc);
+	       TW = FNMS(KP1_647278207, TJ, KP1_018073920 * TI);
+	       TK = FMA(KP1_647278207, TI, KP1_018073920 * TJ);
+	       Td = T9 + Tc;
+	       Tg = FNMS(KP250000000, Td, T6);
+	  }
+	  {
+	       E Tn, TM, Tk, TL;
+	       TP = ii[WS(iis, 2)];
+	       {
+		    E Tl, Tm, Ti, Tj;
+		    Tl = ii[WS(iis, 3)];
+		    Tm = ii[WS(iis, 6)];
+		    Tn = Tl - Tm;
+		    TM = Tl + Tm;
+		    Ti = ii[0];
+		    Tj = ii[WS(iis, 5)];
+		    Tk = Ti + Tj;
+		    TL = Ti - Tj;
+	       }
+	       To = FMA(KP951056516, Tk, KP587785252 * Tn);
+	       TN = KP968245836 * (TL - TM);
+	       TA = FNMS(KP587785252, Tk, KP951056516 * Tn);
+	       TO = TL + TM;
+	       TQ = FMA(KP433012701, TO, KP1_732050807 * TP);
+	  }
+	  T12 = KP1_732050807 * (TP - TO);
+	  Te = T6 + Td;
+	  T11 = Te - T5;
+	  O[0] = FMA(KP2_000000000, Te, T5);
+	  O[WS(os, 10)] = T12 - T11;
+	  O[WS(os, 5)] = T11 + T12;
+	  {
+	       E TE, TG, TB, TF, TY, T10, Tz, TX, TV, TZ;
+	       TE = TC - TD;
+	       TG = TC + TD;
+	       Tz = Tg - Tf;
+	       TB = Tz + TA;
+	       TF = TA - Tz;
+	       TX = TN + TQ;
+	       TY = TW - TX;
+	       T10 = TW + TX;
+	       O[WS(os, 12)] = FMA(KP2_000000000, TB, TE);
+	       O[WS(os, 3)] = FMS(KP2_000000000, TF, TG);
+	       TV = TE - TB;
+	       O[WS(os, 2)] = TV + TY;
+	       O[WS(os, 7)] = TY - TV;
+	       TZ = TF + TG;
+	       O[WS(os, 8)] = TZ - T10;
+	       O[WS(os, 13)] = -(TZ + T10);
+	  }
+	  {
+	       E Tw, Ty, Tp, Tx, TS, TU, Th, TR, TH, TT;
+	       Tw = Ts - Tv;
+	       Ty = Ts + Tv;
+	       Th = Tf + Tg;
+	       Tp = Th + To;
+	       Tx = Th - To;
+	       TR = TN - TQ;
+	       TS = TK + TR;
+	       TU = TR - TK;
+	       O[WS(os, 9)] = -(FMA(KP2_000000000, Tp, Tw));
+	       O[WS(os, 6)] = FMA(KP2_000000000, Tx, Ty);
+	       TH = Tx - Ty;
+	       O[WS(os, 11)] = TH - TS;
+	       O[WS(os, 1)] = TH + TS;
+	       TT = Tw - Tp;
+	       O[WS(os, 4)] = TT - TU;
+	       O[WS(os, 14)] = TT + TU;
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 15, "hc2rIII_15", {49, 11, 15, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_15) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_15, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_16.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_16.c
new file mode 100644
index 0000000..a81a4a1
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_16.c
@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:12:13 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 16 -name hc2rIII_16 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 66 FP additions, 32 FP multiplications,
+ * (or, 54 additions, 20 multiplications, 12 fused multiply/add),
+ * 40 stack variables, and 32 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_16.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_16.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_16.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_16(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_961570560, +1.961570560806460898252364472268478073947867462);
+     DK(KP390180644, +0.390180644032256535696569736954044481855383236);
+     DK(KP1_111140466, +1.111140466039204449485661627897065748749874382);
+     DK(KP1_662939224, +1.662939224605090474157576755235811513477121624);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T7, TW, T13, Tj, TD, TK, TP, TH, Te, TX, T12, To, Tt, Tx, TS;
+	  E Tw, TT, TY;
+	  {
+	       E T3, Tf, TC, TV, T6, Tz, Ti, TU;
+	       {
+		    E T1, T2, TA, TB;
+		    T1 = ri[0];
+		    T2 = ri[WS(ris, 7)];
+		    T3 = T1 + T2;
+		    Tf = T1 - T2;
+		    TA = ii[0];
+		    TB = ii[WS(iis, 7)];
+		    TC = TA + TB;
+		    TV = TB - TA;
+	       }
+	       {
+		    E T4, T5, Tg, Th;
+		    T4 = ri[WS(ris, 4)];
+		    T5 = ri[WS(ris, 3)];
+		    T6 = T4 + T5;
+		    Tz = T4 - T5;
+		    Tg = ii[WS(iis, 4)];
+		    Th = ii[WS(iis, 3)];
+		    Ti = Tg + Th;
+		    TU = Tg - Th;
+	       }
+	       T7 = T3 + T6;
+	       TW = TU + TV;
+	       T13 = TV - TU;
+	       Tj = Tf - Ti;
+	       TD = Tz + TC;
+	       TK = Tz - TC;
+	       TP = T3 - T6;
+	       TH = Tf + Ti;
+	  }
+	  {
+	       E Ta, Tk, Tn, TR, Td, Tp, Ts, TQ;
+	       {
+		    E T8, T9, Tl, Tm;
+		    T8 = ri[WS(ris, 2)];
+		    T9 = ri[WS(ris, 5)];
+		    Ta = T8 + T9;
+		    Tk = T8 - T9;
+		    Tl = ii[WS(iis, 2)];
+		    Tm = ii[WS(iis, 5)];
+		    Tn = Tl + Tm;
+		    TR = Tl - Tm;
+	       }
+	       {
+		    E Tb, Tc, Tq, Tr;
+		    Tb = ri[WS(ris, 1)];
+		    Tc = ri[WS(ris, 6)];
+		    Td = Tb + Tc;
+		    Tp = Tb - Tc;
+		    Tq = ii[WS(iis, 1)];
+		    Tr = ii[WS(iis, 6)];
+		    Ts = Tq + Tr;
+		    TQ = Tr - Tq;
+	       }
+	       Te = Ta + Td;
+	       TX = Ta - Td;
+	       T12 = TR + TQ;
+	       To = Tk - Tn;
+	       Tt = Tp - Ts;
+	       Tx = Tp + Ts;
+	       TS = TQ - TR;
+	       Tw = Tk + Tn;
+	  }
+	  O[0] = KP2_000000000 * (T7 + Te);
+	  O[WS(os, 8)] = KP2_000000000 * (T13 - T12);
+	  TT = TP + TS;
+	  TY = TW - TX;
+	  O[WS(os, 2)] = FMA(KP1_847759065, TT, KP765366864 * TY);
+	  O[WS(os, 10)] = FNMS(KP765366864, TT, KP1_847759065 * TY);
+	  {
+	       E T11, T14, TZ, T10;
+	       T11 = T7 - Te;
+	       T14 = T12 + T13;
+	       O[WS(os, 4)] = KP1_414213562 * (T11 + T14);
+	       O[WS(os, 12)] = KP1_414213562 * (T14 - T11);
+	       TZ = TP - TS;
+	       T10 = TX + TW;
+	       O[WS(os, 6)] = FMA(KP765366864, TZ, KP1_847759065 * T10);
+	       O[WS(os, 14)] = FNMS(KP1_847759065, TZ, KP765366864 * T10);
+	  }
+	  {
+	       E TJ, TN, TM, TO, TI, TL;
+	       TI = KP707106781 * (Tw + Tx);
+	       TJ = TH - TI;
+	       TN = TH + TI;
+	       TL = KP707106781 * (To - Tt);
+	       TM = TK - TL;
+	       TO = TL + TK;
+	       O[WS(os, 3)] = FMA(KP1_662939224, TJ, KP1_111140466 * TM);
+	       O[WS(os, 15)] = FNMS(KP1_961570560, TN, KP390180644 * TO);
+	       O[WS(os, 11)] = FNMS(KP1_111140466, TJ, KP1_662939224 * TM);
+	       O[WS(os, 7)] = FMA(KP390180644, TN, KP1_961570560 * TO);
+	  }
+	  {
+	       E Tv, TF, TE, TG, Tu, Ty;
+	       Tu = KP707106781 * (To + Tt);
+	       Tv = Tj + Tu;
+	       TF = Tj - Tu;
+	       Ty = KP707106781 * (Tw - Tx);
+	       TE = Ty + TD;
+	       TG = Ty - TD;
+	       O[WS(os, 1)] = FNMS(KP390180644, TE, KP1_961570560 * Tv);
+	       O[WS(os, 13)] = FNMS(KP1_662939224, TF, KP1_111140466 * TG);
+	       O[WS(os, 9)] = -(FMA(KP390180644, Tv, KP1_961570560 * TE));
+	       O[WS(os, 5)] = FMA(KP1_111140466, TF, KP1_662939224 * TG);
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 16, "hc2rIII_16", {54, 20, 12, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_16) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_16, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_2.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_2.c
new file mode 100644
index 0000000..c01ed70
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_2.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:54 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 2 -name hc2rIII_2 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 0 FP additions, 2 FP multiplications,
+ * (or, 0 additions, 2 multiplications, 0 fused multiply/add),
+ * 4 stack variables, and 4 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_2.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_2.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_2.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_2(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T1, T2;
+	  T1 = ri[0];
+	  O[0] = KP2_000000000 * T1;
+	  T2 = ii[0];
+	  O[WS(os, 1)] = -(KP2_000000000 * T2);
+     }
+}
+
+static const khc2r_desc desc = { 2, "hc2rIII_2", {0, 2, 0, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_2) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_2, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_3.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_3.c
new file mode 100644
index 0000000..05b761d
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_3.c
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:54 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 3 -name hc2rIII_3 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 4 FP additions, 2 FP multiplications,
+ * (or, 3 additions, 1 multiplications, 1 fused multiply/add),
+ * 8 stack variables, and 6 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_3.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_3.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_3.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_3(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T5, T1, T2, T3, T4;
+	  T4 = ii[0];
+	  T5 = KP1_732050807 * T4;
+	  T1 = ri[WS(ris, 1)];
+	  T2 = ri[0];
+	  T3 = T2 - T1;
+	  O[0] = FMA(KP2_000000000, T2, T1);
+	  O[WS(os, 2)] = -(T3 + T5);
+	  O[WS(os, 1)] = T3 - T5;
+     }
+}
+
+static const khc2r_desc desc = { 3, "hc2rIII_3", {3, 1, 1, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_3) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_3, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_32.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_32.c
new file mode 100644
index 0000000..ab9409b
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_32.c
@@ -0,0 +1,371 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:12:16 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 32 -name hc2rIII_32 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 174 FP additions, 84 FP multiplications,
+ * (or, 138 additions, 48 multiplications, 36 fused multiply/add),
+ * 66 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_32(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_913880671, +1.913880671464417729871595773960539938965698411);
+     DK(KP580569354, +0.580569354508924735272384751634790549382952557);
+     DK(KP942793473, +0.942793473651995297112775251810508755314920638);
+     DK(KP1_763842528, +1.763842528696710059425513727320776699016885241);
+     DK(KP1_546020906, +1.546020906725473921621813219516939601942082586);
+     DK(KP1_268786568, +1.268786568327290996430343226450986741351374190);
+     DK(KP196034280, +0.196034280659121203988391127777283691722273346);
+     DK(KP1_990369453, +1.990369453344393772489673906218959843150949737);
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     DK(KP1_961570560, +1.961570560806460898252364472268478073947867462);
+     DK(KP390180644, +0.390180644032256535696569736954044481855383236);
+     DK(KP1_111140466, +1.111140466039204449485661627897065748749874382);
+     DK(KP1_662939224, +1.662939224605090474157576755235811513477121624);
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T7, T2i, T2F, Tz, T1k, T1I, T1Z, T1x, Te, T22, T2E, T2j, T1f, T1y, TK;
+	  E T1J, Tm, T2B, TW, T1a, T1C, T1L, T28, T2l, Tt, T2A, T17, T1b, T1F, T1M;
+	  E T2d, T2m;
+	  {
+	       E T3, Tv, T1j, T2h, T6, T1g, Ty, T2g;
+	       {
+		    E T1, T2, T1h, T1i;
+		    T1 = ri[0];
+		    T2 = ri[WS(ris, 15)];
+		    T3 = T1 + T2;
+		    Tv = T1 - T2;
+		    T1h = ii[0];
+		    T1i = ii[WS(iis, 15)];
+		    T1j = T1h + T1i;
+		    T2h = T1i - T1h;
+	       }
+	       {
+		    E T4, T5, Tw, Tx;
+		    T4 = ri[WS(ris, 8)];
+		    T5 = ri[WS(ris, 7)];
+		    T6 = T4 + T5;
+		    T1g = T4 - T5;
+		    Tw = ii[WS(iis, 8)];
+		    Tx = ii[WS(iis, 7)];
+		    Ty = Tw + Tx;
+		    T2g = Tw - Tx;
+	       }
+	       T7 = T3 + T6;
+	       T2i = T2g + T2h;
+	       T2F = T2h - T2g;
+	       Tz = Tv - Ty;
+	       T1k = T1g + T1j;
+	       T1I = T1g - T1j;
+	       T1Z = T3 - T6;
+	       T1x = Tv + Ty;
+	  }
+	  {
+	       E Ta, TA, TD, T21, Td, TF, TI, T20;
+	       {
+		    E T8, T9, TB, TC;
+		    T8 = ri[WS(ris, 4)];
+		    T9 = ri[WS(ris, 11)];
+		    Ta = T8 + T9;
+		    TA = T8 - T9;
+		    TB = ii[WS(iis, 4)];
+		    TC = ii[WS(iis, 11)];
+		    TD = TB + TC;
+		    T21 = TB - TC;
+	       }
+	       {
+		    E Tb, Tc, TG, TH;
+		    Tb = ri[WS(ris, 3)];
+		    Tc = ri[WS(ris, 12)];
+		    Td = Tb + Tc;
+		    TF = Tb - Tc;
+		    TG = ii[WS(iis, 3)];
+		    TH = ii[WS(iis, 12)];
+		    TI = TG + TH;
+		    T20 = TH - TG;
+	       }
+	       Te = Ta + Td;
+	       T22 = T20 - T21;
+	       T2E = T21 + T20;
+	       T2j = Ta - Td;
+	       {
+		    E T1d, T1e, TE, TJ;
+		    T1d = TA + TD;
+		    T1e = TF + TI;
+		    T1f = KP707106781 * (T1d - T1e);
+		    T1y = KP707106781 * (T1d + T1e);
+		    TE = TA - TD;
+		    TJ = TF - TI;
+		    TK = KP707106781 * (TE + TJ);
+		    T1J = KP707106781 * (TE - TJ);
+	       }
+	  }
+	  {
+	       E Ti, TM, TU, T25, Tl, TR, TP, T26, TQ, TV;
+	       {
+		    E Tg, Th, TS, TT;
+		    Tg = ri[WS(ris, 2)];
+		    Th = ri[WS(ris, 13)];
+		    Ti = Tg + Th;
+		    TM = Tg - Th;
+		    TS = ii[WS(iis, 2)];
+		    TT = ii[WS(iis, 13)];
+		    TU = TS + TT;
+		    T25 = TS - TT;
+	       }
+	       {
+		    E Tj, Tk, TN, TO;
+		    Tj = ri[WS(ris, 10)];
+		    Tk = ri[WS(ris, 5)];
+		    Tl = Tj + Tk;
+		    TR = Tj - Tk;
+		    TN = ii[WS(iis, 10)];
+		    TO = ii[WS(iis, 5)];
+		    TP = TN + TO;
+		    T26 = TN - TO;
+	       }
+	       Tm = Ti + Tl;
+	       T2B = T26 + T25;
+	       TQ = TM - TP;
+	       TV = TR + TU;
+	       TW = FNMS(KP382683432, TV, KP923879532 * TQ);
+	       T1a = FMA(KP382683432, TQ, KP923879532 * TV);
+	       {
+		    E T1A, T1B, T24, T27;
+		    T1A = TM + TP;
+		    T1B = TU - TR;
+		    T1C = FNMS(KP923879532, T1B, KP382683432 * T1A);
+		    T1L = FMA(KP923879532, T1A, KP382683432 * T1B);
+		    T24 = Ti - Tl;
+		    T27 = T25 - T26;
+		    T28 = T24 - T27;
+		    T2l = T24 + T27;
+	       }
+	  }
+	  {
+	       E Tp, TX, T15, T2a, Ts, T12, T10, T2b, T11, T16;
+	       {
+		    E Tn, To, T13, T14;
+		    Tn = ri[WS(ris, 1)];
+		    To = ri[WS(ris, 14)];
+		    Tp = Tn + To;
+		    TX = Tn - To;
+		    T13 = ii[WS(iis, 1)];
+		    T14 = ii[WS(iis, 14)];
+		    T15 = T13 + T14;
+		    T2a = T14 - T13;
+	       }
+	       {
+		    E Tq, Tr, TY, TZ;
+		    Tq = ri[WS(ris, 6)];
+		    Tr = ri[WS(ris, 9)];
+		    Ts = Tq + Tr;
+		    T12 = Tq - Tr;
+		    TY = ii[WS(iis, 6)];
+		    TZ = ii[WS(iis, 9)];
+		    T10 = TY + TZ;
+		    T2b = TY - TZ;
+	       }
+	       Tt = Tp + Ts;
+	       T2A = T2b + T2a;
+	       T11 = TX - T10;
+	       T16 = T12 - T15;
+	       T17 = FMA(KP923879532, T11, KP382683432 * T16);
+	       T1b = FNMS(KP382683432, T11, KP923879532 * T16);
+	       {
+		    E T1D, T1E, T29, T2c;
+		    T1D = TX + T10;
+		    T1E = T12 + T15;
+		    T1F = FNMS(KP923879532, T1E, KP382683432 * T1D);
+		    T1M = FMA(KP923879532, T1D, KP382683432 * T1E);
+		    T29 = Tp - Ts;
+		    T2c = T2a - T2b;
+		    T2d = T29 + T2c;
+		    T2m = T2c - T29;
+	       }
+	  }
+	  {
+	       E Tf, Tu, T2L, T2M, T2N, T2O;
+	       Tf = T7 + Te;
+	       Tu = Tm + Tt;
+	       T2L = Tf - Tu;
+	       T2M = T2B + T2A;
+	       T2N = T2F - T2E;
+	       T2O = T2M + T2N;
+	       O[0] = KP2_000000000 * (Tf + Tu);
+	       O[WS(os, 16)] = KP2_000000000 * (T2N - T2M);
+	       O[WS(os, 8)] = KP1_414213562 * (T2L + T2O);
+	       O[WS(os, 24)] = KP1_414213562 * (T2O - T2L);
+	  }
+	  {
+	       E T2t, T2x, T2w, T2y;
+	       {
+		    E T2r, T2s, T2u, T2v;
+		    T2r = T1Z - T22;
+		    T2s = KP707106781 * (T2m - T2l);
+		    T2t = T2r + T2s;
+		    T2x = T2r - T2s;
+		    T2u = T2j + T2i;
+		    T2v = KP707106781 * (T28 - T2d);
+		    T2w = T2u - T2v;
+		    T2y = T2v + T2u;
+	       }
+	       O[WS(os, 6)] = FMA(KP1_662939224, T2t, KP1_111140466 * T2w);
+	       O[WS(os, 30)] = FNMS(KP1_961570560, T2x, KP390180644 * T2y);
+	       O[WS(os, 22)] = FNMS(KP1_111140466, T2t, KP1_662939224 * T2w);
+	       O[WS(os, 14)] = FMA(KP390180644, T2x, KP1_961570560 * T2y);
+	  }
+	  {
+	       E T2D, T2J, T2I, T2K;
+	       {
+		    E T2z, T2C, T2G, T2H;
+		    T2z = T7 - Te;
+		    T2C = T2A - T2B;
+		    T2D = T2z + T2C;
+		    T2J = T2z - T2C;
+		    T2G = T2E + T2F;
+		    T2H = Tm - Tt;
+		    T2I = T2G - T2H;
+		    T2K = T2H + T2G;
+	       }
+	       O[WS(os, 4)] = FMA(KP1_847759065, T2D, KP765366864 * T2I);
+	       O[WS(os, 28)] = FNMS(KP1_847759065, T2J, KP765366864 * T2K);
+	       O[WS(os, 20)] = FNMS(KP765366864, T2D, KP1_847759065 * T2I);
+	       O[WS(os, 12)] = FMA(KP765366864, T2J, KP1_847759065 * T2K);
+	  }
+	  {
+	       E T19, T1n, T1m, T1o;
+	       {
+		    E TL, T18, T1c, T1l;
+		    TL = Tz + TK;
+		    T18 = TW + T17;
+		    T19 = TL + T18;
+		    T1n = TL - T18;
+		    T1c = T1a + T1b;
+		    T1l = T1f + T1k;
+		    T1m = T1c + T1l;
+		    T1o = T1c - T1l;
+	       }
+	       O[WS(os, 1)] = FNMS(KP196034280, T1m, KP1_990369453 * T19);
+	       O[WS(os, 25)] = FNMS(KP1_546020906, T1n, KP1_268786568 * T1o);
+	       O[WS(os, 17)] = -(FMA(KP196034280, T19, KP1_990369453 * T1m));
+	       O[WS(os, 9)] = FMA(KP1_268786568, T1n, KP1_546020906 * T1o);
+	  }
+	  {
+	       E T1r, T1v, T1u, T1w;
+	       {
+		    E T1p, T1q, T1s, T1t;
+		    T1p = Tz - TK;
+		    T1q = T1b - T1a;
+		    T1r = T1p + T1q;
+		    T1v = T1p - T1q;
+		    T1s = T1f - T1k;
+		    T1t = TW - T17;
+		    T1u = T1s - T1t;
+		    T1w = T1t + T1s;
+	       }
+	       O[WS(os, 5)] = FMA(KP1_763842528, T1r, KP942793473 * T1u);
+	       O[WS(os, 29)] = FNMS(KP1_913880671, T1v, KP580569354 * T1w);
+	       O[WS(os, 21)] = FNMS(KP942793473, T1r, KP1_763842528 * T1u);
+	       O[WS(os, 13)] = FMA(KP580569354, T1v, KP1_913880671 * T1w);
+	  }
+	  {
+	       E T1T, T1X, T1W, T1Y;
+	       {
+		    E T1R, T1S, T1U, T1V;
+		    T1R = T1x + T1y;
+		    T1S = T1L + T1M;
+		    T1T = T1R - T1S;
+		    T1X = T1R + T1S;
+		    T1U = T1J + T1I;
+		    T1V = T1C - T1F;
+		    T1W = T1U - T1V;
+		    T1Y = T1V + T1U;
+	       }
+	       O[WS(os, 7)] = FMA(KP1_546020906, T1T, KP1_268786568 * T1W);
+	       O[WS(os, 31)] = FNMS(KP1_990369453, T1X, KP196034280 * T1Y);
+	       O[WS(os, 23)] = FNMS(KP1_268786568, T1T, KP1_546020906 * T1W);
+	       O[WS(os, 15)] = FMA(KP196034280, T1X, KP1_990369453 * T1Y);
+	  }
+	  {
+	       E T2f, T2p, T2o, T2q;
+	       {
+		    E T23, T2e, T2k, T2n;
+		    T23 = T1Z + T22;
+		    T2e = KP707106781 * (T28 + T2d);
+		    T2f = T23 + T2e;
+		    T2p = T23 - T2e;
+		    T2k = T2i - T2j;
+		    T2n = KP707106781 * (T2l + T2m);
+		    T2o = T2k - T2n;
+		    T2q = T2n + T2k;
+	       }
+	       O[WS(os, 2)] = FMA(KP1_961570560, T2f, KP390180644 * T2o);
+	       O[WS(os, 26)] = FNMS(KP1_662939224, T2p, KP1_111140466 * T2q);
+	       O[WS(os, 18)] = FNMS(KP390180644, T2f, KP1_961570560 * T2o);
+	       O[WS(os, 10)] = FMA(KP1_111140466, T2p, KP1_662939224 * T2q);
+	  }
+	  {
+	       E T1H, T1P, T1O, T1Q;
+	       {
+		    E T1z, T1G, T1K, T1N;
+		    T1z = T1x - T1y;
+		    T1G = T1C + T1F;
+		    T1H = T1z + T1G;
+		    T1P = T1z - T1G;
+		    T1K = T1I - T1J;
+		    T1N = T1L - T1M;
+		    T1O = T1K - T1N;
+		    T1Q = T1N + T1K;
+	       }
+	       O[WS(os, 3)] = FMA(KP1_913880671, T1H, KP580569354 * T1O);
+	       O[WS(os, 27)] = FNMS(KP1_763842528, T1P, KP942793473 * T1Q);
+	       O[WS(os, 19)] = FNMS(KP580569354, T1H, KP1_913880671 * T1O);
+	       O[WS(os, 11)] = FMA(KP942793473, T1P, KP1_763842528 * T1Q);
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 32, "hc2rIII_32", {138, 48, 36, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_32) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_32, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_4.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_4.c
new file mode 100644
index 0000000..d562add
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_4.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:54 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 4 -name hc2rIII_4 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 6 FP additions, 4 FP multiplications,
+ * (or, 6 additions, 4 multiplications, 0 fused multiply/add),
+ * 9 stack variables, and 8 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_4.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_4.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_4.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_4(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T1, T2, T3, T4, T5, T6;
+	  T1 = ri[0];
+	  T2 = ri[WS(ris, 1)];
+	  T3 = T1 - T2;
+	  T4 = ii[0];
+	  T5 = ii[WS(iis, 1)];
+	  T6 = T4 + T5;
+	  O[0] = KP2_000000000 * (T1 + T2);
+	  O[WS(os, 2)] = KP2_000000000 * (T5 - T4);
+	  O[WS(os, 1)] = KP1_414213562 * (T3 - T6);
+	  O[WS(os, 3)] = -(KP1_414213562 * (T3 + T6));
+     }
+}
+
+static const khc2r_desc desc = { 4, "hc2rIII_4", {6, 4, 0, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_4) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_4, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_5.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_5.c
new file mode 100644
index 0000000..4fb772d
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_5.c
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:55 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 5 -name hc2rIII_5 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 12 FP additions, 7 FP multiplications,
+ * (or, 8 additions, 3 multiplications, 4 fused multiply/add),
+ * 18 stack variables, and 10 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_5.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_5.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_5.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_5(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_118033988, +1.118033988749894848204586834365638117720309180);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP1_175570504, +1.175570504584946258337411909278145537195304875);
+     DK(KP1_902113032, +1.902113032590307144232878666758764286811397268);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E Ta, Tc, T1, T4, T5, T6, Tb, T7;
+	  {
+	       E T8, T9, T2, T3;
+	       T8 = ii[WS(iis, 1)];
+	       T9 = ii[0];
+	       Ta = FMA(KP1_902113032, T8, KP1_175570504 * T9);
+	       Tc = FNMS(KP1_902113032, T9, KP1_175570504 * T8);
+	       T1 = ri[WS(ris, 2)];
+	       T2 = ri[WS(ris, 1)];
+	       T3 = ri[0];
+	       T4 = T2 + T3;
+	       T5 = FMS(KP500000000, T4, T1);
+	       T6 = KP1_118033988 * (T3 - T2);
+	  }
+	  O[0] = FMA(KP2_000000000, T4, T1);
+	  Tb = T6 - T5;
+	  O[WS(os, 2)] = Tb + Tc;
+	  O[WS(os, 3)] = Tc - Tb;
+	  T7 = T5 + T6;
+	  O[WS(os, 1)] = T7 - Ta;
+	  O[WS(os, 4)] = -(T7 + Ta);
+     }
+}
+
+static const khc2r_desc desc = { 5, "hc2rIII_5", {8, 3, 4, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_5) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_5, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_6.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_6.c
new file mode 100644
index 0000000..506bffd
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_6.c
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:58 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 6 -name hc2rIII_6 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 12 FP additions, 6 FP multiplications,
+ * (or, 10 additions, 4 multiplications, 2 fused multiply/add),
+ * 15 stack variables, and 12 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_6.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_6.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_6.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_6(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T1, T6, T4, T5, T9, Tb, Ta, Tc;
+	  T1 = ri[WS(ris, 1)];
+	  T6 = ii[WS(iis, 1)];
+	  {
+	       E T2, T3, T7, T8;
+	       T2 = ri[WS(ris, 2)];
+	       T3 = ri[0];
+	       T4 = T2 + T3;
+	       T5 = KP1_732050807 * (T2 - T3);
+	       T7 = ii[WS(iis, 2)];
+	       T8 = ii[0];
+	       T9 = T7 + T8;
+	       Tb = KP1_732050807 * (T7 - T8);
+	  }
+	  O[0] = KP2_000000000 * (T1 + T4);
+	  O[WS(os, 3)] = KP2_000000000 * (T6 - T9);
+	  Ta = FMA(KP2_000000000, T6, T9);
+	  O[WS(os, 1)] = -(T5 + Ta);
+	  O[WS(os, 5)] = T5 - Ta;
+	  Tc = FMS(KP2_000000000, T1, T4);
+	  O[WS(os, 2)] = Tb - Tc;
+	  O[WS(os, 4)] = Tc + Tb;
+     }
+}
+
+static const khc2r_desc desc = { 6, "hc2rIII_6", {10, 4, 2, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_6) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_6, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_7.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_7.c
new file mode 100644
index 0000000..f65cbfd
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_7.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:12:00 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 7 -name hc2rIII_7 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 24 FP additions, 19 FP multiplications,
+ * (or, 9 additions, 4 multiplications, 15 fused multiply/add),
+ * 21 stack variables, and 14 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_7.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_7.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_7.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_7(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_246979603, +1.246979603717467061050009768008479621264549462);
+     DK(KP1_801937735, +1.801937735804838252472204639014890102331838324);
+     DK(KP445041867, +0.445041867912628808577805128993589518932711138);
+     DK(KP867767478, +0.867767478235116240951536665696717509219981456);
+     DK(KP1_949855824, +1.949855824363647214036263365987862434465571601);
+     DK(KP1_563662964, +1.563662964936059617416889053348115500464669037);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T9, Td, Tb, T1, T4, T2, T3, T5, Tc, Ta, T6, T8, T7;
+	  T6 = ii[WS(iis, 2)];
+	  T8 = ii[0];
+	  T7 = ii[WS(iis, 1)];
+	  T9 = FMA(KP1_563662964, T6, KP1_949855824 * T7) + (KP867767478 * T8);
+	  Td = FNMS(KP1_949855824, T8, KP1_563662964 * T7) - (KP867767478 * T6);
+	  Tb = FNMS(KP1_563662964, T8, KP1_949855824 * T6) - (KP867767478 * T7);
+	  T1 = ri[WS(ris, 3)];
+	  T4 = ri[0];
+	  T2 = ri[WS(ris, 2)];
+	  T3 = ri[WS(ris, 1)];
+	  T5 = FMA(KP445041867, T3, KP1_801937735 * T4) + FNMA(KP1_246979603, T2, T1);
+	  Tc = FMA(KP1_801937735, T2, KP445041867 * T4) + FNMA(KP1_246979603, T3, T1);
+	  Ta = FMA(KP1_246979603, T4, T1) + FNMA(KP1_801937735, T3, KP445041867 * T2);
+	  O[WS(os, 1)] = T5 - T9;
+	  O[WS(os, 6)] = -(T5 + T9);
+	  O[WS(os, 4)] = Td - Tc;
+	  O[WS(os, 3)] = Tc + Td;
+	  O[WS(os, 5)] = Tb - Ta;
+	  O[WS(os, 2)] = Ta + Tb;
+	  O[0] = FMA(KP2_000000000, T2 + T3 + T4, T1);
+     }
+}
+
+static const khc2r_desc desc = { 7, "hc2rIII_7", {9, 4, 15, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_7) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_7, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_8.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_8.c
new file mode 100644
index 0000000..3a7ffb5
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_8.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:12:01 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 8 -name hc2rIII_8 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 22 FP additions, 12 FP multiplications,
+ * (or, 18 additions, 8 multiplications, 4 fused multiply/add),
+ * 19 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_8.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_8.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_8.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_8(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T3, T7, Tf, Tl, T6, Tc, Ta, Tk, Tb, Tg;
+	  {
+	       E T1, T2, Td, Te;
+	       T1 = ri[0];
+	       T2 = ri[WS(ris, 3)];
+	       T3 = T1 + T2;
+	       T7 = T1 - T2;
+	       Td = ii[0];
+	       Te = ii[WS(iis, 3)];
+	       Tf = Td + Te;
+	       Tl = Te - Td;
+	  }
+	  {
+	       E T4, T5, T8, T9;
+	       T4 = ri[WS(ris, 2)];
+	       T5 = ri[WS(ris, 1)];
+	       T6 = T4 + T5;
+	       Tc = T4 - T5;
+	       T8 = ii[WS(iis, 2)];
+	       T9 = ii[WS(iis, 1)];
+	       Ta = T8 + T9;
+	       Tk = T8 - T9;
+	  }
+	  O[0] = KP2_000000000 * (T3 + T6);
+	  O[WS(os, 4)] = KP2_000000000 * (Tl - Tk);
+	  Tb = T7 - Ta;
+	  Tg = Tc + Tf;
+	  O[WS(os, 1)] = FNMS(KP765366864, Tg, KP1_847759065 * Tb);
+	  O[WS(os, 5)] = -(FMA(KP765366864, Tb, KP1_847759065 * Tg));
+	  {
+	       E Th, Ti, Tj, Tm;
+	       Th = T7 + Ta;
+	       Ti = Tc - Tf;
+	       O[WS(os, 3)] = FMA(KP765366864, Th, KP1_847759065 * Ti);
+	       O[WS(os, 7)] = FNMS(KP1_847759065, Th, KP765366864 * Ti);
+	       Tj = T3 - T6;
+	       Tm = Tk + Tl;
+	       O[WS(os, 2)] = KP1_414213562 * (Tj + Tm);
+	       O[WS(os, 6)] = KP1_414213562 * (Tm - Tj);
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 8, "hc2rIII_8", {18, 8, 4, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_8) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_8, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2rIII_9.c b/src/fftw3/rdft/codelets/hc2r/hc2rIII_9.c
new file mode 100644
index 0000000..0bcc0ea
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2rIII_9.c
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:12:05 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 9 -name hc2rIII_9 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 32 FP additions, 18 FP multiplications,
+ * (or, 22 additions, 8 multiplications, 10 fused multiply/add),
+ * 35 stack variables, and 18 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2rIII_9.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_9.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2rIII_9.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void hc2rIII_9(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP642787609, +0.642787609686539326322643409907263432907559884);
+     DK(KP766044443, +0.766044443118978035202392650555416673935832457);
+     DK(KP1_326827896, +1.326827896337876792410842639271782594433726619);
+     DK(KP1_113340798, +1.113340798452838732905825904094046265936583811);
+     DK(KP984807753, +0.984807753012208059366743024589523013670643252);
+     DK(KP173648177, +0.173648177666930348851716626769314796000375677);
+     DK(KP1_705737063, +1.705737063904886419256501927880148143872040591);
+     DK(KP300767466, +0.300767466360870593278543795225003852144476517);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T3, Ts, Ti, Td, Tc, T8, To, Tu, Tl, Tt, T9, Te;
+	  {
+	       E Th, T1, T2, Tf, Tg;
+	       Tg = ii[WS(iis, 1)];
+	       Th = KP1_732050807 * Tg;
+	       T1 = ri[WS(ris, 4)];
+	       T2 = ri[WS(ris, 1)];
+	       Tf = T2 - T1;
+	       T3 = FMA(KP2_000000000, T2, T1);
+	       Ts = Tf - Th;
+	       Ti = Tf + Th;
+	  }
+	  {
+	       E T4, T7, Tm, Tk, Tn, Tj;
+	       T4 = ri[WS(ris, 3)];
+	       Td = ii[WS(iis, 3)];
+	       {
+		    E T5, T6, Ta, Tb;
+		    T5 = ri[0];
+		    T6 = ri[WS(ris, 2)];
+		    T7 = T5 + T6;
+		    Tm = KP866025403 * (T6 - T5);
+		    Ta = ii[WS(iis, 2)];
+		    Tb = ii[0];
+		    Tc = Ta - Tb;
+		    Tk = KP866025403 * (Tb + Ta);
+	       }
+	       T8 = T4 + T7;
+	       Tn = FMA(KP500000000, Tc, Td);
+	       To = Tm - Tn;
+	       Tu = Tm + Tn;
+	       Tj = FMS(KP500000000, T7, T4);
+	       Tl = Tj + Tk;
+	       Tt = Tj - Tk;
+	  }
+	  O[0] = FMA(KP2_000000000, T8, T3);
+	  T9 = T8 - T3;
+	  Te = KP1_732050807 * (Tc - Td);
+	  O[WS(os, 3)] = T9 + Te;
+	  O[WS(os, 6)] = Te - T9;
+	  {
+	       E Tr, Tp, Tq, Tx, Tv, Tw;
+	       Tr = FNMS(KP1_705737063, Tl, KP300767466 * To);
+	       Tp = FMA(KP173648177, Tl, KP984807753 * To);
+	       Tq = Ti - Tp;
+	       O[WS(os, 2)] = -(FMA(KP2_000000000, Tp, Ti));
+	       O[WS(os, 8)] = Tr - Tq;
+	       O[WS(os, 5)] = Tq + Tr;
+	       Tx = FMA(KP1_113340798, Tt, KP1_326827896 * Tu);
+	       Tv = FNMS(KP642787609, Tu, KP766044443 * Tt);
+	       Tw = Tv - Ts;
+	       O[WS(os, 1)] = FMA(KP2_000000000, Tv, Ts);
+	       O[WS(os, 7)] = Tx - Tw;
+	       O[WS(os, 4)] = Tw + Tx;
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 9, "hc2rIII_9", {22, 8, 10, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2rIII_9) (planner *p) {
+     X(khc2rIII_register) (p, hc2rIII_9, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_10.c b/src/fftw3/rdft/codelets/hc2r/hc2r_10.c
new file mode 100644
index 0000000..9e6c5bc
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_10.c
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 10 -name hc2r_10 -include hc2r.h */
+
+/*
+ * This function contains 34 FP additions, 14 FP multiplications,
+ * (or, 26 additions, 6 multiplications, 8 fused multiply/add),
+ * 26 stack variables, and 20 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_10.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_10.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_10.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_10(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP1_902113032, +1.902113032590307144232878666758764286811397268);
+     DK(KP1_175570504, +1.175570504584946258337411909278145537195304875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_118033988, +1.118033988749894848204586834365638117720309180);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T3, Tb, Tn, Tv, Tk, Tu, Ta, Ts, Te, Tg, Ti, Tj;
+	  {
+	       E T1, T2, Tl, Tm;
+	       T1 = ri[0];
+	       T2 = ri[WS(ris, 5)];
+	       T3 = T1 - T2;
+	       Tb = T1 + T2;
+	       Tl = ii[WS(iis, 4)];
+	       Tm = ii[WS(iis, 1)];
+	       Tn = Tl - Tm;
+	       Tv = Tl + Tm;
+	  }
+	  Ti = ii[WS(iis, 2)];
+	  Tj = ii[WS(iis, 3)];
+	  Tk = Ti - Tj;
+	  Tu = Ti + Tj;
+	  {
+	       E T6, Tc, T9, Td;
+	       {
+		    E T4, T5, T7, T8;
+		    T4 = ri[WS(ris, 2)];
+		    T5 = ri[WS(ris, 3)];
+		    T6 = T4 - T5;
+		    Tc = T4 + T5;
+		    T7 = ri[WS(ris, 4)];
+		    T8 = ri[WS(ris, 1)];
+		    T9 = T7 - T8;
+		    Td = T7 + T8;
+	       }
+	       Ta = T6 + T9;
+	       Ts = KP1_118033988 * (T6 - T9);
+	       Te = Tc + Td;
+	       Tg = KP1_118033988 * (Tc - Td);
+	  }
+	  O[WS(os, 5)] = FMA(KP2_000000000, Ta, T3);
+	  O[0] = FMA(KP2_000000000, Te, Tb);
+	  {
+	       E To, Tq, Th, Tp, Tf;
+	       To = FNMS(KP1_902113032, Tn, KP1_175570504 * Tk);
+	       Tq = FMA(KP1_902113032, Tk, KP1_175570504 * Tn);
+	       Tf = FNMS(KP500000000, Te, Tb);
+	       Th = Tf - Tg;
+	       Tp = Tg + Tf;
+	       O[WS(os, 2)] = Th - To;
+	       O[WS(os, 4)] = Tp + Tq;
+	       O[WS(os, 8)] = Th + To;
+	       O[WS(os, 6)] = Tp - Tq;
+	  }
+	  {
+	       E Tw, Ty, Tt, Tx, Tr;
+	       Tw = FNMS(KP1_902113032, Tv, KP1_175570504 * Tu);
+	       Ty = FMA(KP1_902113032, Tu, KP1_175570504 * Tv);
+	       Tr = FNMS(KP500000000, Ta, T3);
+	       Tt = Tr - Ts;
+	       Tx = Ts + Tr;
+	       O[WS(os, 7)] = Tt - Tw;
+	       O[WS(os, 9)] = Tx + Ty;
+	       O[WS(os, 3)] = Tt + Tw;
+	       O[WS(os, 1)] = Tx - Ty;
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 10, "hc2r_10", {26, 6, 8, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_10) (planner *p) {
+     X(khc2r_register) (p, hc2r_10, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_11.c b/src/fftw3/rdft/codelets/hc2r/hc2r_11.c
new file mode 100644
index 0000000..2310502
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_11.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 11 -name hc2r_11 -include hc2r.h */
+
+/*
+ * This function contains 60 FP additions, 51 FP multiplications,
+ * (or, 19 additions, 10 multiplications, 41 fused multiply/add),
+ * 33 stack variables, and 22 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_11.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_11.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_11.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_11(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_918985947, +1.918985947228994779780736114132655398124909697);
+     DK(KP1_309721467, +1.309721467890570128113850144932587106367582399);
+     DK(KP284629676, +0.284629676546570280887585337232739337582102722);
+     DK(KP830830026, +0.830830026003772851058548298459246407048009821);
+     DK(KP1_682507065, +1.682507065662362337723623297838735435026584997);
+     DK(KP563465113, +0.563465113682859395422835830693233798071555798);
+     DK(KP1_511499148, +1.511499148708516567548071687944688840359434890);
+     DK(KP1_979642883, +1.979642883761865464752184075553437574753038744);
+     DK(KP1_819263990, +1.819263990709036742823430766158056920120482102);
+     DK(KP1_081281634, +1.081281634911195164215271908637383390863541216);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E Td, Tl, Tf, Th, Tj, T1, T2, T6, T5, T4, T3, T7, Tk, Te, Tg;
+	  E Ti;
+	  {
+	       E T8, Tc, T9, Ta, Tb;
+	       T8 = ii[WS(iis, 2)];
+	       Tc = ii[WS(iis, 1)];
+	       T9 = ii[WS(iis, 4)];
+	       Ta = ii[WS(iis, 5)];
+	       Tb = ii[WS(iis, 3)];
+	       Td = FMA(KP1_081281634, T8, KP1_819263990 * T9) + FNMA(KP1_979642883, Ta, KP1_511499148 * Tb) - (KP563465113 * Tc);
+	       Tl = FMA(KP1_979642883, T8, KP1_819263990 * Ta) + FNMA(KP563465113, T9, KP1_081281634 * Tb) - (KP1_511499148 * Tc);
+	       Tf = FMA(KP563465113, T8, KP1_819263990 * Tb) + FNMA(KP1_511499148, Ta, KP1_081281634 * T9) - (KP1_979642883 * Tc);
+	       Th = FMA(KP1_081281634, Tc, KP1_819263990 * T8) + FMA(KP1_979642883, Tb, KP1_511499148 * T9) + (KP563465113 * Ta);
+	       Tj = FMA(KP563465113, Tb, KP1_979642883 * T9) + FNMS(KP1_511499148, T8, KP1_081281634 * Ta) - (KP1_819263990 * Tc);
+	  }
+	  T1 = ri[0];
+	  T2 = ri[WS(ris, 1)];
+	  T6 = ri[WS(ris, 5)];
+	  T5 = ri[WS(ris, 4)];
+	  T4 = ri[WS(ris, 3)];
+	  T3 = ri[WS(ris, 2)];
+	  T7 = FMA(KP1_682507065, T3, T1) + FNMS(KP284629676, T6, KP830830026 * T5) + FNMA(KP1_309721467, T4, KP1_918985947 * T2);
+	  Tk = FMA(KP1_682507065, T4, T1) + FNMS(KP1_918985947, T5, KP830830026 * T6) + FNMA(KP284629676, T3, KP1_309721467 * T2);
+	  Te = FMA(KP830830026, T4, T1) + FNMS(KP1_309721467, T6, KP1_682507065 * T5) + FNMA(KP1_918985947, T3, KP284629676 * T2);
+	  Tg = FMA(KP1_682507065, T2, T1) + FNMS(KP1_918985947, T6, KP830830026 * T3) + FNMA(KP1_309721467, T5, KP284629676 * T4);
+	  Ti = FMA(KP830830026, T2, T1) + FNMS(KP284629676, T5, KP1_682507065 * T6) + FNMA(KP1_918985947, T4, KP1_309721467 * T3);
+	  O[WS(os, 6)] = T7 - Td;
+	  O[WS(os, 8)] = Te - Tf;
+	  O[WS(os, 4)] = Tk + Tl;
+	  O[WS(os, 5)] = T7 + Td;
+	  O[WS(os, 7)] = Tk - Tl;
+	  O[WS(os, 2)] = Ti + Tj;
+	  O[WS(os, 3)] = Te + Tf;
+	  O[WS(os, 10)] = Tg + Th;
+	  O[WS(os, 1)] = Tg - Th;
+	  O[WS(os, 9)] = Ti - Tj;
+	  O[0] = FMA(KP2_000000000, T2 + T3 + T4 + T5 + T6, T1);
+     }
+}
+
+static const khc2r_desc desc = { 11, "hc2r_11", {19, 10, 41, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_11) (planner *p) {
+     X(khc2r_register) (p, hc2r_11, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_12.c b/src/fftw3/rdft/codelets/hc2r/hc2r_12.c
new file mode 100644
index 0000000..4fc3710
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_12.c
@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 12 -name hc2r_12 -include hc2r.h */
+
+/*
+ * This function contains 38 FP additions, 10 FP multiplications,
+ * (or, 34 additions, 6 multiplications, 4 fused multiply/add),
+ * 25 stack variables, and 24 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_12.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_12.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_12.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_12(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T8, Tb, Tm, TA, Tw, Tx, Tp, TB, T3, Tr, Tg, T6, Ts, Tk;
+	  {
+	       E T9, Ta, Tn, To;
+	       T8 = ri[WS(ris, 3)];
+	       T9 = ri[WS(ris, 5)];
+	       Ta = ri[WS(ris, 1)];
+	       Tb = T9 + Ta;
+	       Tm = FMS(KP2_000000000, T8, Tb);
+	       TA = KP1_732050807 * (T9 - Ta);
+	       Tw = ii[WS(iis, 3)];
+	       Tn = ii[WS(iis, 5)];
+	       To = ii[WS(iis, 1)];
+	       Tx = Tn + To;
+	       Tp = KP1_732050807 * (Tn - To);
+	       TB = FMA(KP2_000000000, Tw, Tx);
+	  }
+	  {
+	       E Tf, T1, T2, Td, Te;
+	       Te = ii[WS(iis, 4)];
+	       Tf = KP1_732050807 * Te;
+	       T1 = ri[0];
+	       T2 = ri[WS(ris, 4)];
+	       Td = T1 - T2;
+	       T3 = FMA(KP2_000000000, T2, T1);
+	       Tr = Td - Tf;
+	       Tg = Td + Tf;
+	  }
+	  {
+	       E Tj, T4, T5, Th, Ti;
+	       Ti = ii[WS(iis, 2)];
+	       Tj = KP1_732050807 * Ti;
+	       T4 = ri[WS(ris, 6)];
+	       T5 = ri[WS(ris, 2)];
+	       Th = T4 - T5;
+	       T6 = FMA(KP2_000000000, T5, T4);
+	       Ts = Th + Tj;
+	       Tk = Th - Tj;
+	  }
+	  {
+	       E T7, Tc, Tz, TC;
+	       T7 = T3 + T6;
+	       Tc = KP2_000000000 * (T8 + Tb);
+	       O[WS(os, 6)] = T7 - Tc;
+	       O[0] = T7 + Tc;
+	       {
+		    E Tl, Tq, TD, TE;
+		    Tl = Tg + Tk;
+		    Tq = Tm - Tp;
+		    O[WS(os, 2)] = Tl - Tq;
+		    O[WS(os, 8)] = Tl + Tq;
+		    TD = Tg - Tk;
+		    TE = TB - TA;
+		    O[WS(os, 5)] = TD - TE;
+		    O[WS(os, 11)] = TD + TE;
+	       }
+	       Tz = Tr - Ts;
+	       TC = TA + TB;
+	       O[WS(os, 1)] = Tz - TC;
+	       O[WS(os, 7)] = Tz + TC;
+	       {
+		    E Tv, Ty, Tt, Tu;
+		    Tv = T3 - T6;
+		    Ty = KP2_000000000 * (Tw - Tx);
+		    O[WS(os, 9)] = Tv - Ty;
+		    O[WS(os, 3)] = Tv + Ty;
+		    Tt = Tr + Ts;
+		    Tu = Tm + Tp;
+		    O[WS(os, 10)] = Tt - Tu;
+		    O[WS(os, 4)] = Tt + Tu;
+	       }
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 12, "hc2r_12", {34, 6, 4, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_12) (planner *p) {
+     X(khc2r_register) (p, hc2r_12, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_13.c b/src/fftw3/rdft/codelets/hc2r/hc2r_13.c
new file mode 100644
index 0000000..3e1ccbd
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_13.c
@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:12 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 13 -name hc2r_13 -include hc2r.h */
+
+/*
+ * This function contains 76 FP additions, 35 FP multiplications,
+ * (or, 56 additions, 15 multiplications, 20 fused multiply/add),
+ * 56 stack variables, and 26 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_13.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_13.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_13.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_13(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_007074065, +1.007074065727533254493747707736933954186697125);
+     DK(KP227708958, +0.227708958111581597949308691735310621069285120);
+     DK(KP531932498, +0.531932498429674575175042127684371897596660533);
+     DK(KP774781170, +0.774781170935234584261351932853525703557550433);
+     DK(KP265966249, +0.265966249214837287587521063842185948798330267);
+     DK(KP516520780, +0.516520780623489722840901288569017135705033622);
+     DK(KP151805972, +0.151805972074387731966205794490207080712856746);
+     DK(KP503537032, +0.503537032863766627246873853868466977093348562);
+     DK(KP166666666, +0.166666666666666666666666666666666666666666667);
+     DK(KP600925212, +0.600925212577331548853203544578415991041882762);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP256247671, +0.256247671582936600958684654061725059144125175);
+     DK(KP156891391, +0.156891391051584611046832726756003269660212636);
+     DK(KP348277202, +0.348277202304271810011321589858529485233929352);
+     DK(KP1_150281458, +1.150281458948006242736771094910906776922003215);
+     DK(KP300238635, +0.300238635966332641462884626667381504676006424);
+     DK(KP011599105, +0.011599105605768290721655456654083252189827041);
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E TG, TS, TR, T15, TJ, TT, T1, Tm, Tc, Td, Tg, Tj, Tk, Tn, To;
+	  E Tp;
+	  {
+	       E Ts, Tv, Tw, TE, TC, TB, Tz, TD, TA, TF;
+	       {
+		    E Tt, Tu, Tx, Ty;
+		    Ts = ii[WS(iis, 1)];
+		    Tt = ii[WS(iis, 3)];
+		    Tu = ii[WS(iis, 4)];
+		    Tv = Tt - Tu;
+		    Tw = FMS(KP2_000000000, Ts, Tv);
+		    TE = KP1_732050807 * (Tt + Tu);
+		    TC = ii[WS(iis, 5)];
+		    Tx = ii[WS(iis, 6)];
+		    Ty = ii[WS(iis, 2)];
+		    TB = Tx + Ty;
+		    Tz = KP1_732050807 * (Tx - Ty);
+		    TD = FNMS(KP2_000000000, TC, TB);
+	       }
+	       TA = Tw + Tz;
+	       TF = TD - TE;
+	       TG = FMA(KP011599105, TA, KP300238635 * TF);
+	       TS = FNMS(KP011599105, TF, KP300238635 * TA);
+	       {
+		    E TP, TQ, TH, TI;
+		    TP = Ts + Tv;
+		    TQ = TB + TC;
+		    TR = FNMS(KP348277202, TQ, KP1_150281458 * TP);
+		    T15 = FMA(KP348277202, TP, KP1_150281458 * TQ);
+		    TH = Tw - Tz;
+		    TI = TE + TD;
+		    TJ = FMA(KP156891391, TH, KP256247671 * TI);
+		    TT = FNMS(KP256247671, TH, KP156891391 * TI);
+	       }
+	  }
+	  {
+	       E Tb, Ti, Tf, T6, Th, Te;
+	       T1 = ri[0];
+	       {
+		    E T7, T8, T9, Ta;
+		    T7 = ri[WS(ris, 5)];
+		    T8 = ri[WS(ris, 2)];
+		    T9 = ri[WS(ris, 6)];
+		    Ta = T8 + T9;
+		    Tb = T7 + Ta;
+		    Ti = FNMS(KP500000000, Ta, T7);
+		    Tf = T8 - T9;
+	       }
+	       {
+		    E T2, T3, T4, T5;
+		    T2 = ri[WS(ris, 1)];
+		    T3 = ri[WS(ris, 3)];
+		    T4 = ri[WS(ris, 4)];
+		    T5 = T3 + T4;
+		    T6 = T2 + T5;
+		    Th = FNMS(KP500000000, T5, T2);
+		    Te = T3 - T4;
+	       }
+	       Tm = KP600925212 * (T6 - Tb);
+	       Tc = T6 + Tb;
+	       Td = FNMS(KP166666666, Tc, T1);
+	       Tg = Te + Tf;
+	       Tj = Th + Ti;
+	       Tk = FMA(KP503537032, Tg, KP151805972 * Tj);
+	       Tn = Th - Ti;
+	       To = Te - Tf;
+	       Tp = FNMS(KP265966249, To, KP516520780 * Tn);
+	  }
+	  O[0] = FMA(KP2_000000000, Tc, T1);
+	  {
+	       E TK, T1b, TV, T12, T16, T18, TO, T1a, Tr, T17, T11, T13;
+	       {
+		    E TU, T14, TM, TN;
+		    TK = KP1_732050807 * (TG + TJ);
+		    T1b = KP1_732050807 * (TS - TT);
+		    TU = TS + TT;
+		    TV = TR - TU;
+		    T12 = FMA(KP2_000000000, TU, TR);
+		    T14 = TG - TJ;
+		    T16 = FMS(KP2_000000000, T14, T15);
+		    T18 = T14 + T15;
+		    TM = FMA(KP774781170, To, KP531932498 * Tn);
+		    TN = FNMS(KP1_007074065, Tj, KP227708958 * Tg);
+		    TO = TM - TN;
+		    T1a = TM + TN;
+		    {
+			 E Tl, Tq, TZ, T10;
+			 Tl = Td - Tk;
+			 Tq = Tm - Tp;
+			 Tr = Tl - Tq;
+			 T17 = Tq + Tl;
+			 TZ = FMA(KP2_000000000, Tk, Td);
+			 T10 = FMA(KP2_000000000, Tp, Tm);
+			 T11 = TZ - T10;
+			 T13 = T10 + TZ;
+		    }
+	       }
+	       O[WS(os, 5)] = T11 - T12;
+	       O[WS(os, 12)] = T13 - T16;
+	       O[WS(os, 1)] = T13 + T16;
+	       O[WS(os, 8)] = T11 + T12;
+	       {
+		    E TL, TW, T19, T1c;
+		    TL = Tr - TK;
+		    TW = TO - TV;
+		    O[WS(os, 7)] = TL - TW;
+		    O[WS(os, 2)] = TL + TW;
+		    T19 = T17 - T18;
+		    T1c = T1a + T1b;
+		    O[WS(os, 3)] = T19 - T1c;
+		    O[WS(os, 9)] = T1c + T19;
+	       }
+	       {
+		    E T1d, T1e, TX, TY;
+		    T1d = T1a - T1b;
+		    T1e = T17 + T18;
+		    O[WS(os, 4)] = T1d + T1e;
+		    O[WS(os, 10)] = T1e - T1d;
+		    TX = Tr + TK;
+		    TY = TO + TV;
+		    O[WS(os, 6)] = TX - TY;
+		    O[WS(os, 11)] = TX + TY;
+	       }
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 13, "hc2r_13", {56, 15, 20, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_13) (planner *p) {
+     X(khc2r_register) (p, hc2r_13, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_14.c b/src/fftw3/rdft/codelets/hc2r/hc2r_14.c
new file mode 100644
index 0000000..dbbb22e
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_14.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:12 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 14 -name hc2r_14 -include hc2r.h */
+
+/*
+ * This function contains 62 FP additions, 38 FP multiplications,
+ * (or, 36 additions, 12 multiplications, 26 fused multiply/add),
+ * 28 stack variables, and 28 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_14.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_14.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_14.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_14(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_801937735, +1.801937735804838252472204639014890102331838324);
+     DK(KP445041867, +0.445041867912628808577805128993589518932711138);
+     DK(KP1_246979603, +1.246979603717467061050009768008479621264549462);
+     DK(KP867767478, +0.867767478235116240951536665696717509219981456);
+     DK(KP1_949855824, +1.949855824363647214036263365987862434465571601);
+     DK(KP1_563662964, +1.563662964936059617416889053348115500464669037);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T3, Td, T6, Te, Tq, Tz, Tn, Ty, Tc, Tg, Tk, Tx, T9, Tf, T1;
+	  E T2;
+	  T1 = ri[0];
+	  T2 = ri[WS(ris, 7)];
+	  T3 = T1 - T2;
+	  Td = T1 + T2;
+	  {
+	       E T4, T5, To, Tp;
+	       T4 = ri[WS(ris, 2)];
+	       T5 = ri[WS(ris, 5)];
+	       T6 = T4 - T5;
+	       Te = T4 + T5;
+	       To = ii[WS(iis, 2)];
+	       Tp = ii[WS(iis, 5)];
+	       Tq = To - Tp;
+	       Tz = To + Tp;
+	  }
+	  {
+	       E Tl, Tm, Ta, Tb;
+	       Tl = ii[WS(iis, 6)];
+	       Tm = ii[WS(iis, 1)];
+	       Tn = Tl - Tm;
+	       Ty = Tl + Tm;
+	       Ta = ri[WS(ris, 6)];
+	       Tb = ri[WS(ris, 1)];
+	       Tc = Ta - Tb;
+	       Tg = Ta + Tb;
+	  }
+	  {
+	       E Ti, Tj, T7, T8;
+	       Ti = ii[WS(iis, 4)];
+	       Tj = ii[WS(iis, 3)];
+	       Tk = Ti - Tj;
+	       Tx = Ti + Tj;
+	       T7 = ri[WS(ris, 4)];
+	       T8 = ri[WS(ris, 3)];
+	       T9 = T7 - T8;
+	       Tf = T7 + T8;
+	  }
+	  O[WS(os, 7)] = FMA(KP2_000000000, T6 + T9 + Tc, T3);
+	  O[0] = FMA(KP2_000000000, Te + Tf + Tg, Td);
+	  {
+	       E Tr, Th, TE, TD;
+	       Tr = FNMS(KP1_949855824, Tn, KP1_563662964 * Tk) - (KP867767478 * Tq);
+	       Th = FMA(KP1_246979603, Tf, Td) + FNMA(KP445041867, Tg, KP1_801937735 * Te);
+	       O[WS(os, 4)] = Th - Tr;
+	       O[WS(os, 10)] = Th + Tr;
+	       TE = FMA(KP867767478, Tx, KP1_563662964 * Ty) - (KP1_949855824 * Tz);
+	       TD = FMA(KP1_246979603, Tc, T3) + FNMA(KP1_801937735, T9, KP445041867 * T6);
+	       O[WS(os, 5)] = TD - TE;
+	       O[WS(os, 9)] = TD + TE;
+	  }
+	  {
+	       E Tt, Ts, TA, Tw;
+	       Tt = FMA(KP867767478, Tk, KP1_563662964 * Tn) - (KP1_949855824 * Tq);
+	       Ts = FMA(KP1_246979603, Tg, Td) + FNMA(KP1_801937735, Tf, KP445041867 * Te);
+	       O[WS(os, 12)] = Ts - Tt;
+	       O[WS(os, 2)] = Ts + Tt;
+	       TA = FNMS(KP1_949855824, Ty, KP1_563662964 * Tx) - (KP867767478 * Tz);
+	       Tw = FMA(KP1_246979603, T9, T3) + FNMA(KP445041867, Tc, KP1_801937735 * T6);
+	       O[WS(os, 11)] = Tw - TA;
+	       O[WS(os, 3)] = Tw + TA;
+	  }
+	  {
+	       E TC, TB, Tv, Tu;
+	       TC = FMA(KP1_563662964, Tz, KP1_949855824 * Tx) + (KP867767478 * Ty);
+	       TB = FMA(KP1_246979603, T6, T3) + FNMA(KP1_801937735, Tc, KP445041867 * T9);
+	       O[WS(os, 1)] = TB - TC;
+	       O[WS(os, 13)] = TB + TC;
+	       Tv = FMA(KP1_563662964, Tq, KP1_949855824 * Tk) + (KP867767478 * Tn);
+	       Tu = FMA(KP1_246979603, Te, Td) + FNMA(KP1_801937735, Tg, KP445041867 * Tf);
+	       O[WS(os, 8)] = Tu - Tv;
+	       O[WS(os, 6)] = Tu + Tv;
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 14, "hc2r_14", {36, 12, 26, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_14) (planner *p) {
+     X(khc2r_register) (p, hc2r_14, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_15.c b/src/fftw3/rdft/codelets/hc2r/hc2r_15.c
new file mode 100644
index 0000000..d2eed35
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_15.c
@@ -0,0 +1,161 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:12 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 15 -name hc2r_15 -include hc2r.h */
+
+/*
+ * This function contains 64 FP additions, 31 FP multiplications,
+ * (or, 47 additions, 14 multiplications, 17 fused multiply/add),
+ * 44 stack variables, and 30 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_15.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_15.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_15.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_15(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_118033988, +1.118033988749894848204586834365638117720309180);
+     DK(KP1_902113032, +1.902113032590307144232878666758764286811397268);
+     DK(KP1_175570504, +1.175570504584946258337411909278145537195304875);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T3, Tu, Ti, TB, TZ, T10, TE, TG, TJ, Tn, Tv, Ts, Tw, T8, Td;
+	  E Te;
+	  {
+	       E Th, T1, T2, Tf, Tg;
+	       Tg = ii[WS(iis, 5)];
+	       Th = KP1_732050807 * Tg;
+	       T1 = ri[0];
+	       T2 = ri[WS(ris, 5)];
+	       Tf = T1 - T2;
+	       T3 = FMA(KP2_000000000, T2, T1);
+	       Tu = Tf - Th;
+	       Ti = Tf + Th;
+	  }
+	  {
+	       E T4, TD, T9, TI, T5, T6, T7, Ta, Tb, Tc, Tr, TH, Tm, TC, Tj;
+	       E To;
+	       T4 = ri[WS(ris, 3)];
+	       TD = ii[WS(iis, 3)];
+	       T9 = ri[WS(ris, 6)];
+	       TI = ii[WS(iis, 6)];
+	       T5 = ri[WS(ris, 7)];
+	       T6 = ri[WS(ris, 2)];
+	       T7 = T5 + T6;
+	       Ta = ri[WS(ris, 4)];
+	       Tb = ri[WS(ris, 1)];
+	       Tc = Ta + Tb;
+	       {
+		    E Tp, Tq, Tk, Tl;
+		    Tp = ii[WS(iis, 4)];
+		    Tq = ii[WS(iis, 1)];
+		    Tr = KP866025403 * (Tp + Tq);
+		    TH = Tp - Tq;
+		    Tk = ii[WS(iis, 7)];
+		    Tl = ii[WS(iis, 2)];
+		    Tm = KP866025403 * (Tk - Tl);
+		    TC = Tk + Tl;
+	       }
+	       TB = KP866025403 * (T5 - T6);
+	       TZ = TD - TC;
+	       T10 = TI - TH;
+	       TE = FMA(KP500000000, TC, TD);
+	       TG = KP866025403 * (Ta - Tb);
+	       TJ = FMA(KP500000000, TH, TI);
+	       Tj = FNMS(KP500000000, T7, T4);
+	       Tn = Tj - Tm;
+	       Tv = Tj + Tm;
+	       To = FNMS(KP500000000, Tc, T9);
+	       Ts = To - Tr;
+	       Tw = To + Tr;
+	       T8 = T4 + T7;
+	       Td = T9 + Tc;
+	       Te = T8 + Td;
+	  }
+	  O[0] = FMA(KP2_000000000, Te, T3);
+	  {
+	       E T11, T13, TY, T12, TW, TX;
+	       T11 = FNMS(KP1_902113032, T10, KP1_175570504 * TZ);
+	       T13 = FMA(KP1_902113032, TZ, KP1_175570504 * T10);
+	       TW = FNMS(KP500000000, Te, T3);
+	       TX = KP1_118033988 * (T8 - Td);
+	       TY = TW - TX;
+	       T12 = TX + TW;
+	       O[WS(os, 12)] = TY - T11;
+	       O[WS(os, 9)] = T12 + T13;
+	       O[WS(os, 3)] = TY + T11;
+	       O[WS(os, 6)] = T12 - T13;
+	  }
+	  {
+	       E TP, Tt, TO, TT, TV, TR, TS, TU, TQ;
+	       TP = KP1_118033988 * (Tn - Ts);
+	       Tt = Tn + Ts;
+	       TO = FNMS(KP500000000, Tt, Ti);
+	       TR = TE - TB;
+	       TS = TJ - TG;
+	       TT = FNMS(KP1_902113032, TS, KP1_175570504 * TR);
+	       TV = FMA(KP1_902113032, TR, KP1_175570504 * TS);
+	       O[WS(os, 5)] = FMA(KP2_000000000, Tt, Ti);
+	       TU = TP + TO;
+	       O[WS(os, 11)] = TU - TV;
+	       O[WS(os, 14)] = TU + TV;
+	       TQ = TO - TP;
+	       O[WS(os, 2)] = TQ - TT;
+	       O[WS(os, 8)] = TQ + TT;
+	  }
+	  {
+	       E Tz, Tx, Ty, TL, TN, TF, TK, TM, TA;
+	       Tz = KP1_118033988 * (Tv - Tw);
+	       Tx = Tv + Tw;
+	       Ty = FNMS(KP500000000, Tx, Tu);
+	       TF = TB + TE;
+	       TK = TG + TJ;
+	       TL = FNMS(KP1_902113032, TK, KP1_175570504 * TF);
+	       TN = FMA(KP1_902113032, TF, KP1_175570504 * TK);
+	       O[WS(os, 10)] = FMA(KP2_000000000, Tx, Tu);
+	       TM = Tz + Ty;
+	       O[WS(os, 1)] = TM - TN;
+	       O[WS(os, 4)] = TM + TN;
+	       TA = Ty - Tz;
+	       O[WS(os, 7)] = TA - TL;
+	       O[WS(os, 13)] = TA + TL;
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 15, "hc2r_15", {47, 14, 17, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_15) (planner *p) {
+     X(khc2r_register) (p, hc2r_15, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_16.c b/src/fftw3/rdft/codelets/hc2r/hc2r_16.c
new file mode 100644
index 0000000..60ca018
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_16.c
@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:13 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 16 -name hc2r_16 -include hc2r.h */
+
+/*
+ * This function contains 58 FP additions, 18 FP multiplications,
+ * (or, 54 additions, 14 multiplications, 4 fused multiply/add),
+ * 31 stack variables, and 32 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_16.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_16.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_16.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_16(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T9, TS, Tl, TG, T6, TR, Ti, TD, Td, Tq, Tg, Tt, Tn, Tu, TV;
+	  E TU, TN, TK;
+	  {
+	       E T7, T8, TE, Tj, Tk, TF;
+	       T7 = ri[WS(ris, 2)];
+	       T8 = ri[WS(ris, 6)];
+	       TE = T7 - T8;
+	       Tj = ii[WS(iis, 2)];
+	       Tk = ii[WS(iis, 6)];
+	       TF = Tj + Tk;
+	       T9 = KP2_000000000 * (T7 + T8);
+	       TS = KP1_414213562 * (TE + TF);
+	       Tl = KP2_000000000 * (Tj - Tk);
+	       TG = KP1_414213562 * (TE - TF);
+	  }
+	  {
+	       E T5, TC, T3, TA;
+	       {
+		    E T4, TB, T1, T2;
+		    T4 = ri[WS(ris, 4)];
+		    T5 = KP2_000000000 * T4;
+		    TB = ii[WS(iis, 4)];
+		    TC = KP2_000000000 * TB;
+		    T1 = ri[0];
+		    T2 = ri[WS(ris, 8)];
+		    T3 = T1 + T2;
+		    TA = T1 - T2;
+	       }
+	       T6 = T3 + T5;
+	       TR = TA + TC;
+	       Ti = T3 - T5;
+	       TD = TA - TC;
+	  }
+	  {
+	       E TI, TM, TL, TJ;
+	       {
+		    E Tb, Tc, To, Tp;
+		    Tb = ri[WS(ris, 1)];
+		    Tc = ri[WS(ris, 7)];
+		    Td = Tb + Tc;
+		    TI = Tb - Tc;
+		    To = ii[WS(iis, 1)];
+		    Tp = ii[WS(iis, 7)];
+		    Tq = To - Tp;
+		    TM = To + Tp;
+	       }
+	       {
+		    E Te, Tf, Tr, Ts;
+		    Te = ri[WS(ris, 5)];
+		    Tf = ri[WS(ris, 3)];
+		    Tg = Te + Tf;
+		    TL = Te - Tf;
+		    Tr = ii[WS(iis, 5)];
+		    Ts = ii[WS(iis, 3)];
+		    Tt = Tr - Ts;
+		    TJ = Tr + Ts;
+	       }
+	       Tn = Td - Tg;
+	       Tu = Tq - Tt;
+	       TV = TM - TL;
+	       TU = TI + TJ;
+	       TN = TL + TM;
+	       TK = TI - TJ;
+	  }
+	  {
+	       E Ta, Th, TT, TW;
+	       Ta = T6 + T9;
+	       Th = KP2_000000000 * (Td + Tg);
+	       O[WS(os, 8)] = Ta - Th;
+	       O[0] = Ta + Th;
+	       TT = TR - TS;
+	       TW = FNMS(KP1_847759065, TV, KP765366864 * TU);
+	       O[WS(os, 11)] = TT - TW;
+	       O[WS(os, 3)] = TT + TW;
+	  }
+	  {
+	       E TX, TY, Tm, Tv;
+	       TX = TR + TS;
+	       TY = FMA(KP1_847759065, TU, KP765366864 * TV);
+	       O[WS(os, 7)] = TX - TY;
+	       O[WS(os, 15)] = TX + TY;
+	       Tm = Ti - Tl;
+	       Tv = KP1_414213562 * (Tn - Tu);
+	       O[WS(os, 10)] = Tm - Tv;
+	       O[WS(os, 2)] = Tm + Tv;
+	  }
+	  {
+	       E Tw, Tx, TH, TO;
+	       Tw = Ti + Tl;
+	       Tx = KP1_414213562 * (Tn + Tu);
+	       O[WS(os, 6)] = Tw - Tx;
+	       O[WS(os, 14)] = Tw + Tx;
+	       TH = TD + TG;
+	       TO = FNMS(KP765366864, TN, KP1_847759065 * TK);
+	       O[WS(os, 9)] = TH - TO;
+	       O[WS(os, 1)] = TH + TO;
+	  }
+	  {
+	       E TP, TQ, Ty, Tz;
+	       TP = TD - TG;
+	       TQ = FMA(KP765366864, TK, KP1_847759065 * TN);
+	       O[WS(os, 5)] = TP - TQ;
+	       O[WS(os, 13)] = TP + TQ;
+	       Ty = T6 - T9;
+	       Tz = KP2_000000000 * (Tt + Tq);
+	       O[WS(os, 4)] = Ty - Tz;
+	       O[WS(os, 12)] = Ty + Tz;
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 16, "hc2r_16", {54, 14, 4, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_16) (planner *p) {
+     X(khc2r_register) (p, hc2r_16, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_3.c b/src/fftw3/rdft/codelets/hc2r/hc2r_3.c
new file mode 100644
index 0000000..990d625
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_3.c
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 3 -name hc2r_3 -include hc2r.h */
+
+/*
+ * This function contains 4 FP additions, 2 FP multiplications,
+ * (or, 3 additions, 1 multiplications, 1 fused multiply/add),
+ * 8 stack variables, and 6 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_3.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_3.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_3.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_3(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T5, T1, T2, T3, T4;
+	  T4 = ii[WS(iis, 1)];
+	  T5 = KP1_732050807 * T4;
+	  T1 = ri[0];
+	  T2 = ri[WS(ris, 1)];
+	  T3 = T1 - T2;
+	  O[0] = FMA(KP2_000000000, T2, T1);
+	  O[WS(os, 2)] = T3 + T5;
+	  O[WS(os, 1)] = T3 - T5;
+     }
+}
+
+static const khc2r_desc desc = { 3, "hc2r_3", {3, 1, 1, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_3) (planner *p) {
+     X(khc2r_register) (p, hc2r_3, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_32.c b/src/fftw3/rdft/codelets/hc2r/hc2r_32.c
new file mode 100644
index 0000000..6611654
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_32.c
@@ -0,0 +1,336 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:14 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 32 -name hc2r_32 -include hc2r.h */
+
+/*
+ * This function contains 156 FP additions, 50 FP multiplications,
+ * (or, 140 additions, 34 multiplications, 16 fused multiply/add),
+ * 54 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_32(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_662939224, +1.662939224605090474157576755235811513477121624);
+     DK(KP1_111140466, +1.111140466039204449485661627897065748749874382);
+     DK(KP1_961570560, +1.961570560806460898252364472268478073947867462);
+     DK(KP390180644, +0.390180644032256535696569736954044481855383236);
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T9, T2c, TB, T1y, T6, T2b, Ty, T1v, Th, T2e, T2f, TD, TK, T1C, T1F;
+	  E T1h, Tp, T2i, T2m, TN, T13, T1K, T1Y, T1k, Tw, TU, T1l, TW, T1V, T2j;
+	  E T1R, T2l;
+	  {
+	       E T7, T8, T1w, Tz, TA, T1x;
+	       T7 = ri[WS(ris, 4)];
+	       T8 = ri[WS(ris, 12)];
+	       T1w = T7 - T8;
+	       Tz = ii[WS(iis, 4)];
+	       TA = ii[WS(iis, 12)];
+	       T1x = Tz + TA;
+	       T9 = KP2_000000000 * (T7 + T8);
+	       T2c = KP1_414213562 * (T1w + T1x);
+	       TB = KP2_000000000 * (Tz - TA);
+	       T1y = KP1_414213562 * (T1w - T1x);
+	  }
+	  {
+	       E T5, T1u, T3, T1s;
+	       {
+		    E T4, T1t, T1, T2;
+		    T4 = ri[WS(ris, 8)];
+		    T5 = KP2_000000000 * T4;
+		    T1t = ii[WS(iis, 8)];
+		    T1u = KP2_000000000 * T1t;
+		    T1 = ri[0];
+		    T2 = ri[WS(ris, 16)];
+		    T3 = T1 + T2;
+		    T1s = T1 - T2;
+	       }
+	       T6 = T3 + T5;
+	       T2b = T1s + T1u;
+	       Ty = T3 - T5;
+	       T1v = T1s - T1u;
+	  }
+	  {
+	       E Td, T1A, TG, T1E, Tg, T1D, TJ, T1B;
+	       {
+		    E Tb, Tc, TE, TF;
+		    Tb = ri[WS(ris, 2)];
+		    Tc = ri[WS(ris, 14)];
+		    Td = Tb + Tc;
+		    T1A = Tb - Tc;
+		    TE = ii[WS(iis, 2)];
+		    TF = ii[WS(iis, 14)];
+		    TG = TE - TF;
+		    T1E = TE + TF;
+	       }
+	       {
+		    E Te, Tf, TH, TI;
+		    Te = ri[WS(ris, 10)];
+		    Tf = ri[WS(ris, 6)];
+		    Tg = Te + Tf;
+		    T1D = Te - Tf;
+		    TH = ii[WS(iis, 10)];
+		    TI = ii[WS(iis, 6)];
+		    TJ = TH - TI;
+		    T1B = TH + TI;
+	       }
+	       Th = KP2_000000000 * (Td + Tg);
+	       T2e = T1A + T1B;
+	       T2f = T1E - T1D;
+	       TD = Td - Tg;
+	       TK = TG - TJ;
+	       T1C = T1A - T1B;
+	       T1F = T1D + T1E;
+	       T1h = KP2_000000000 * (TJ + TG);
+	  }
+	  {
+	       E Tl, T1I, TZ, T1X, To, T1W, T12, T1J;
+	       {
+		    E Tj, Tk, TX, TY;
+		    Tj = ri[WS(ris, 1)];
+		    Tk = ri[WS(ris, 15)];
+		    Tl = Tj + Tk;
+		    T1I = Tj - Tk;
+		    TX = ii[WS(iis, 1)];
+		    TY = ii[WS(iis, 15)];
+		    TZ = TX - TY;
+		    T1X = TX + TY;
+	       }
+	       {
+		    E Tm, Tn, T10, T11;
+		    Tm = ri[WS(ris, 9)];
+		    Tn = ri[WS(ris, 7)];
+		    To = Tm + Tn;
+		    T1W = Tm - Tn;
+		    T10 = ii[WS(iis, 9)];
+		    T11 = ii[WS(iis, 7)];
+		    T12 = T10 - T11;
+		    T1J = T10 + T11;
+	       }
+	       Tp = Tl + To;
+	       T2i = T1I + T1J;
+	       T2m = T1X - T1W;
+	       TN = Tl - To;
+	       T13 = TZ - T12;
+	       T1K = T1I - T1J;
+	       T1Y = T1W + T1X;
+	       T1k = T12 + TZ;
+	  }
+	  {
+	       E Ts, T1L, TT, T1M, Tv, T1O, TQ, T1P;
+	       {
+		    E Tq, Tr, TR, TS;
+		    Tq = ri[WS(ris, 5)];
+		    Tr = ri[WS(ris, 11)];
+		    Ts = Tq + Tr;
+		    T1L = Tq - Tr;
+		    TR = ii[WS(iis, 5)];
+		    TS = ii[WS(iis, 11)];
+		    TT = TR - TS;
+		    T1M = TR + TS;
+	       }
+	       {
+		    E Tt, Tu, TO, TP;
+		    Tt = ri[WS(ris, 3)];
+		    Tu = ri[WS(ris, 13)];
+		    Tv = Tt + Tu;
+		    T1O = Tt - Tu;
+		    TO = ii[WS(iis, 13)];
+		    TP = ii[WS(iis, 3)];
+		    TQ = TO - TP;
+		    T1P = TP + TO;
+	       }
+	       Tw = Ts + Tv;
+	       TU = TQ - TT;
+	       T1l = TT + TQ;
+	       TW = Ts - Tv;
+	       {
+		    E T1T, T1U, T1N, T1Q;
+		    T1T = T1L + T1M;
+		    T1U = T1O + T1P;
+		    T1V = KP707106781 * (T1T - T1U);
+		    T2j = KP707106781 * (T1T + T1U);
+		    T1N = T1L - T1M;
+		    T1Q = T1O - T1P;
+		    T1R = KP707106781 * (T1N + T1Q);
+		    T2l = KP707106781 * (T1N - T1Q);
+	       }
+	  }
+	  {
+	       E Tx, T1r, Ti, T1q, Ta;
+	       Tx = KP2_000000000 * (Tp + Tw);
+	       T1r = KP2_000000000 * (T1l + T1k);
+	       Ta = T6 + T9;
+	       Ti = Ta + Th;
+	       T1q = Ta - Th;
+	       O[WS(os, 16)] = Ti - Tx;
+	       O[WS(os, 24)] = T1q + T1r;
+	       O[0] = Ti + Tx;
+	       O[WS(os, 8)] = T1q - T1r;
+	  }
+	  {
+	       E T1i, T1o, T1n, T1p, T1g, T1j, T1m;
+	       T1g = T6 - T9;
+	       T1i = T1g - T1h;
+	       T1o = T1g + T1h;
+	       T1j = Tp - Tw;
+	       T1m = T1k - T1l;
+	       T1n = KP1_414213562 * (T1j - T1m);
+	       T1p = KP1_414213562 * (T1j + T1m);
+	       O[WS(os, 20)] = T1i - T1n;
+	       O[WS(os, 28)] = T1o + T1p;
+	       O[WS(os, 4)] = T1i + T1n;
+	       O[WS(os, 12)] = T1o - T1p;
+	  }
+	  {
+	       E TM, T16, T15, T17;
+	       {
+		    E TC, TL, TV, T14;
+		    TC = Ty - TB;
+		    TL = KP1_414213562 * (TD - TK);
+		    TM = TC + TL;
+		    T16 = TC - TL;
+		    TV = TN + TU;
+		    T14 = TW + T13;
+		    T15 = FNMS(KP765366864, T14, KP1_847759065 * TV);
+		    T17 = FMA(KP765366864, TV, KP1_847759065 * T14);
+	       }
+	       O[WS(os, 18)] = TM - T15;
+	       O[WS(os, 26)] = T16 + T17;
+	       O[WS(os, 2)] = TM + T15;
+	       O[WS(os, 10)] = T16 - T17;
+	  }
+	  {
+	       E T2t, T2x, T2w, T2y;
+	       {
+		    E T2r, T2s, T2u, T2v;
+		    T2r = T2b + T2c;
+		    T2s = FMA(KP1_847759065, T2e, KP765366864 * T2f);
+		    T2t = T2r - T2s;
+		    T2x = T2r + T2s;
+		    T2u = T2i + T2j;
+		    T2v = T2m - T2l;
+		    T2w = FNMS(KP1_961570560, T2v, KP390180644 * T2u);
+		    T2y = FMA(KP1_961570560, T2u, KP390180644 * T2v);
+	       }
+	       O[WS(os, 23)] = T2t - T2w;
+	       O[WS(os, 31)] = T2x + T2y;
+	       O[WS(os, 7)] = T2t + T2w;
+	       O[WS(os, 15)] = T2x - T2y;
+	  }
+	  {
+	       E T1a, T1e, T1d, T1f;
+	       {
+		    E T18, T19, T1b, T1c;
+		    T18 = Ty + TB;
+		    T19 = KP1_414213562 * (TD + TK);
+		    T1a = T18 - T19;
+		    T1e = T18 + T19;
+		    T1b = TN - TU;
+		    T1c = T13 - TW;
+		    T1d = FNMS(KP1_847759065, T1c, KP765366864 * T1b);
+		    T1f = FMA(KP1_847759065, T1b, KP765366864 * T1c);
+	       }
+	       O[WS(os, 22)] = T1a - T1d;
+	       O[WS(os, 30)] = T1e + T1f;
+	       O[WS(os, 6)] = T1a + T1d;
+	       O[WS(os, 14)] = T1e - T1f;
+	  }
+	  {
+	       E T25, T29, T28, T2a;
+	       {
+		    E T23, T24, T26, T27;
+		    T23 = T1v - T1y;
+		    T24 = FMA(KP765366864, T1C, KP1_847759065 * T1F);
+		    T25 = T23 - T24;
+		    T29 = T23 + T24;
+		    T26 = T1K - T1R;
+		    T27 = T1Y - T1V;
+		    T28 = FNMS(KP1_662939224, T27, KP1_111140466 * T26);
+		    T2a = FMA(KP1_662939224, T26, KP1_111140466 * T27);
+	       }
+	       O[WS(os, 21)] = T25 - T28;
+	       O[WS(os, 29)] = T29 + T2a;
+	       O[WS(os, 5)] = T25 + T28;
+	       O[WS(os, 13)] = T29 - T2a;
+	  }
+	  {
+	       E T2h, T2p, T2o, T2q;
+	       {
+		    E T2d, T2g, T2k, T2n;
+		    T2d = T2b - T2c;
+		    T2g = FNMS(KP1_847759065, T2f, KP765366864 * T2e);
+		    T2h = T2d + T2g;
+		    T2p = T2d - T2g;
+		    T2k = T2i - T2j;
+		    T2n = T2l + T2m;
+		    T2o = FNMS(KP1_111140466, T2n, KP1_662939224 * T2k);
+		    T2q = FMA(KP1_111140466, T2k, KP1_662939224 * T2n);
+	       }
+	       O[WS(os, 19)] = T2h - T2o;
+	       O[WS(os, 27)] = T2p + T2q;
+	       O[WS(os, 3)] = T2h + T2o;
+	       O[WS(os, 11)] = T2p - T2q;
+	  }
+	  {
+	       E T1H, T21, T20, T22;
+	       {
+		    E T1z, T1G, T1S, T1Z;
+		    T1z = T1v + T1y;
+		    T1G = FNMS(KP765366864, T1F, KP1_847759065 * T1C);
+		    T1H = T1z + T1G;
+		    T21 = T1z - T1G;
+		    T1S = T1K + T1R;
+		    T1Z = T1V + T1Y;
+		    T20 = FNMS(KP390180644, T1Z, KP1_961570560 * T1S);
+		    T22 = FMA(KP390180644, T1S, KP1_961570560 * T1Z);
+	       }
+	       O[WS(os, 17)] = T1H - T20;
+	       O[WS(os, 25)] = T21 + T22;
+	       O[WS(os, 1)] = T1H + T20;
+	       O[WS(os, 9)] = T21 - T22;
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 32, "hc2r_32", {140, 34, 16, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_32) (planner *p) {
+     X(khc2r_register) (p, hc2r_32, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_4.c b/src/fftw3/rdft/codelets/hc2r/hc2r_4.c
new file mode 100644
index 0000000..00db38d
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_4.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 4 -name hc2r_4 -include hc2r.h */
+
+/*
+ * This function contains 6 FP additions, 2 FP multiplications,
+ * (or, 6 additions, 2 multiplications, 0 fused multiply/add),
+ * 10 stack variables, and 8 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_4.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_4.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_4.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_4(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T5, T8, T3, T6;
+	  {
+	       E T4, T7, T1, T2;
+	       T4 = ri[WS(ris, 1)];
+	       T5 = KP2_000000000 * T4;
+	       T7 = ii[WS(iis, 1)];
+	       T8 = KP2_000000000 * T7;
+	       T1 = ri[0];
+	       T2 = ri[WS(ris, 2)];
+	       T3 = T1 + T2;
+	       T6 = T1 - T2;
+	  }
+	  O[WS(os, 2)] = T3 - T5;
+	  O[WS(os, 3)] = T6 + T8;
+	  O[0] = T3 + T5;
+	  O[WS(os, 1)] = T6 - T8;
+     }
+}
+
+static const khc2r_desc desc = { 4, "hc2r_4", {6, 2, 0, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_4) (planner *p) {
+     X(khc2r_register) (p, hc2r_4, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_5.c b/src/fftw3/rdft/codelets/hc2r/hc2r_5.c
new file mode 100644
index 0000000..fd1c09d
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_5.c
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 5 -name hc2r_5 -include hc2r.h */
+
+/*
+ * This function contains 12 FP additions, 7 FP multiplications,
+ * (or, 8 additions, 3 multiplications, 4 fused multiply/add),
+ * 18 stack variables, and 10 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_5.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_5.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_5.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_5(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_118033988, +1.118033988749894848204586834365638117720309180);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP1_902113032, +1.902113032590307144232878666758764286811397268);
+     DK(KP1_175570504, +1.175570504584946258337411909278145537195304875);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E Ta, Tc, T1, T4, T5, T6, Tb, T7;
+	  {
+	       E T8, T9, T2, T3;
+	       T8 = ii[WS(iis, 1)];
+	       T9 = ii[WS(iis, 2)];
+	       Ta = FNMS(KP1_902113032, T9, KP1_175570504 * T8);
+	       Tc = FMA(KP1_902113032, T8, KP1_175570504 * T9);
+	       T1 = ri[0];
+	       T2 = ri[WS(ris, 1)];
+	       T3 = ri[WS(ris, 2)];
+	       T4 = T2 + T3;
+	       T5 = FNMS(KP500000000, T4, T1);
+	       T6 = KP1_118033988 * (T2 - T3);
+	  }
+	  O[0] = FMA(KP2_000000000, T4, T1);
+	  Tb = T6 + T5;
+	  O[WS(os, 1)] = Tb - Tc;
+	  O[WS(os, 4)] = Tb + Tc;
+	  T7 = T5 - T6;
+	  O[WS(os, 2)] = T7 - Ta;
+	  O[WS(os, 3)] = T7 + Ta;
+     }
+}
+
+static const khc2r_desc desc = { 5, "hc2r_5", {8, 3, 4, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_5) (planner *p) {
+     X(khc2r_register) (p, hc2r_5, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_6.c b/src/fftw3/rdft/codelets/hc2r/hc2r_6.c
new file mode 100644
index 0000000..08ce9d0
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_6.c
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 6 -name hc2r_6 -include hc2r.h */
+
+/*
+ * This function contains 14 FP additions, 4 FP multiplications,
+ * (or, 12 additions, 2 multiplications, 2 fused multiply/add),
+ * 17 stack variables, and 12 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_6.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_6.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_6.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_6(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T3, T7, Tc, Te, T6, T8, T1, T2, T9, Td;
+	  T1 = ri[0];
+	  T2 = ri[WS(ris, 3)];
+	  T3 = T1 - T2;
+	  T7 = T1 + T2;
+	  {
+	       E Ta, Tb, T4, T5;
+	       Ta = ii[WS(iis, 2)];
+	       Tb = ii[WS(iis, 1)];
+	       Tc = KP1_732050807 * (Ta - Tb);
+	       Te = KP1_732050807 * (Ta + Tb);
+	       T4 = ri[WS(ris, 2)];
+	       T5 = ri[WS(ris, 1)];
+	       T6 = T4 - T5;
+	       T8 = T4 + T5;
+	  }
+	  O[WS(os, 3)] = FMA(KP2_000000000, T6, T3);
+	  O[0] = FMA(KP2_000000000, T8, T7);
+	  T9 = T7 - T8;
+	  O[WS(os, 4)] = T9 - Tc;
+	  O[WS(os, 2)] = T9 + Tc;
+	  Td = T3 - T6;
+	  O[WS(os, 1)] = Td - Te;
+	  O[WS(os, 5)] = Td + Te;
+     }
+}
+
+static const khc2r_desc desc = { 6, "hc2r_6", {12, 2, 2, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_6) (planner *p) {
+     X(khc2r_register) (p, hc2r_6, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_7.c b/src/fftw3/rdft/codelets/hc2r/hc2r_7.c
new file mode 100644
index 0000000..6d1f4c1
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_7.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 7 -name hc2r_7 -include hc2r.h */
+
+/*
+ * This function contains 24 FP additions, 19 FP multiplications,
+ * (or, 11 additions, 6 multiplications, 13 fused multiply/add),
+ * 21 stack variables, and 14 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_7.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_7.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_7.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_7(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_801937735, +1.801937735804838252472204639014890102331838324);
+     DK(KP445041867, +0.445041867912628808577805128993589518932711138);
+     DK(KP1_246979603, +1.246979603717467061050009768008479621264549462);
+     DK(KP867767478, +0.867767478235116240951536665696717509219981456);
+     DK(KP1_949855824, +1.949855824363647214036263365987862434465571601);
+     DK(KP1_563662964, +1.563662964936059617416889053348115500464669037);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T9, Td, Tb, T1, T4, T2, T3, T5, Tc, Ta, T6, T8, T7;
+	  T6 = ii[WS(iis, 2)];
+	  T8 = ii[WS(iis, 1)];
+	  T7 = ii[WS(iis, 3)];
+	  T9 = FNMS(KP1_949855824, T7, KP1_563662964 * T6) - (KP867767478 * T8);
+	  Td = FMA(KP867767478, T6, KP1_563662964 * T7) - (KP1_949855824 * T8);
+	  Tb = FMA(KP1_563662964, T8, KP1_949855824 * T6) + (KP867767478 * T7);
+	  T1 = ri[0];
+	  T4 = ri[WS(ris, 3)];
+	  T2 = ri[WS(ris, 1)];
+	  T3 = ri[WS(ris, 2)];
+	  T5 = FMA(KP1_246979603, T3, T1) + FNMA(KP445041867, T4, KP1_801937735 * T2);
+	  Tc = FMA(KP1_246979603, T4, T1) + FNMA(KP1_801937735, T3, KP445041867 * T2);
+	  Ta = FMA(KP1_246979603, T2, T1) + FNMA(KP1_801937735, T4, KP445041867 * T3);
+	  O[WS(os, 4)] = T5 - T9;
+	  O[WS(os, 3)] = T5 + T9;
+	  O[WS(os, 2)] = Tc + Td;
+	  O[WS(os, 5)] = Tc - Td;
+	  O[WS(os, 6)] = Ta + Tb;
+	  O[WS(os, 1)] = Ta - Tb;
+	  O[0] = FMA(KP2_000000000, T2 + T3 + T4, T1);
+     }
+}
+
+static const khc2r_desc desc = { 7, "hc2r_7", {11, 6, 13, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_7) (planner *p) {
+     X(khc2r_register) (p, hc2r_7, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_8.c b/src/fftw3/rdft/codelets/hc2r/hc2r_8.c
new file mode 100644
index 0000000..dd11bd4
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_8.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 8 -name hc2r_8 -include hc2r.h */
+
+/*
+ * This function contains 20 FP additions, 6 FP multiplications,
+ * (or, 20 additions, 6 multiplications, 0 fused multiply/add),
+ * 21 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_8.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_8.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_8.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_8(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T5, Tg, T3, Te, T9, Ti, Td, Tj, T6, Ta;
+	  {
+	       E T4, Tf, T1, T2;
+	       T4 = ri[WS(ris, 2)];
+	       T5 = KP2_000000000 * T4;
+	       Tf = ii[WS(iis, 2)];
+	       Tg = KP2_000000000 * Tf;
+	       T1 = ri[0];
+	       T2 = ri[WS(ris, 4)];
+	       T3 = T1 + T2;
+	       Te = T1 - T2;
+	       {
+		    E T7, T8, Tb, Tc;
+		    T7 = ri[WS(ris, 1)];
+		    T8 = ri[WS(ris, 3)];
+		    T9 = KP2_000000000 * (T7 + T8);
+		    Ti = T7 - T8;
+		    Tb = ii[WS(iis, 1)];
+		    Tc = ii[WS(iis, 3)];
+		    Td = KP2_000000000 * (Tb - Tc);
+		    Tj = Tb + Tc;
+	       }
+	  }
+	  T6 = T3 + T5;
+	  O[WS(os, 4)] = T6 - T9;
+	  O[0] = T6 + T9;
+	  Ta = T3 - T5;
+	  O[WS(os, 2)] = Ta - Td;
+	  O[WS(os, 6)] = Ta + Td;
+	  {
+	       E Th, Tk, Tl, Tm;
+	       Th = Te - Tg;
+	       Tk = KP1_414213562 * (Ti - Tj);
+	       O[WS(os, 5)] = Th - Tk;
+	       O[WS(os, 1)] = Th + Tk;
+	       Tl = Te + Tg;
+	       Tm = KP1_414213562 * (Ti + Tj);
+	       O[WS(os, 3)] = Tl - Tm;
+	       O[WS(os, 7)] = Tl + Tm;
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 8, "hc2r_8", {20, 6, 0, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_8) (planner *p) {
+     X(khc2r_register) (p, hc2r_8, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hc2r_9.c b/src/fftw3/rdft/codelets/hc2r/hc2r_9.c
new file mode 100644
index 0000000..91b0f78
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hc2r_9.c
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r -compact -variables 4 -sign 1 -n 9 -name hc2r_9 -include hc2r.h */
+
+/*
+ * This function contains 32 FP additions, 18 FP multiplications,
+ * (or, 22 additions, 8 multiplications, 10 fused multiply/add),
+ * 35 stack variables, and 18 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hc2r_9.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_9.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: hc2r_9.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void hc2r_9(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     DK(KP984807753, +0.984807753012208059366743024589523013670643252);
+     DK(KP173648177, +0.173648177666930348851716626769314796000375677);
+     DK(KP300767466, +0.300767466360870593278543795225003852144476517);
+     DK(KP1_705737063, +1.705737063904886419256501927880148143872040591);
+     DK(KP642787609, +0.642787609686539326322643409907263432907559884);
+     DK(KP766044443, +0.766044443118978035202392650555416673935832457);
+     DK(KP1_326827896, +1.326827896337876792410842639271782594433726619);
+     DK(KP1_113340798, +1.113340798452838732905825904094046265936583811);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     int i;
+     for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, O = O + ovs) {
+	  E T3, Tq, Tc, Tk, Tj, T8, Tm, Ts, Th, Tr, Tw, Tx;
+	  {
+	       E Tb, T1, T2, T9, Ta;
+	       Ta = ii[WS(iis, 3)];
+	       Tb = KP1_732050807 * Ta;
+	       T1 = ri[0];
+	       T2 = ri[WS(ris, 3)];
+	       T9 = T1 - T2;
+	       T3 = FMA(KP2_000000000, T2, T1);
+	       Tq = T9 + Tb;
+	       Tc = T9 - Tb;
+	  }
+	  {
+	       E T4, T7, Ti, Tg, Tl, Td;
+	       T4 = ri[WS(ris, 1)];
+	       Tk = ii[WS(iis, 1)];
+	       {
+		    E T5, T6, Te, Tf;
+		    T5 = ri[WS(ris, 4)];
+		    T6 = ri[WS(ris, 2)];
+		    T7 = T5 + T6;
+		    Ti = KP866025403 * (T5 - T6);
+		    Te = ii[WS(iis, 4)];
+		    Tf = ii[WS(iis, 2)];
+		    Tg = KP866025403 * (Te + Tf);
+		    Tj = Tf - Te;
+	       }
+	       T8 = T4 + T7;
+	       Tl = FMA(KP500000000, Tj, Tk);
+	       Tm = Ti + Tl;
+	       Ts = Tl - Ti;
+	       Td = FNMS(KP500000000, T7, T4);
+	       Th = Td - Tg;
+	       Tr = Td + Tg;
+	  }
+	  O[0] = FMA(KP2_000000000, T8, T3);
+	  Tw = T3 - T8;
+	  Tx = KP1_732050807 * (Tk - Tj);
+	  O[WS(os, 3)] = Tw - Tx;
+	  O[WS(os, 6)] = Tw + Tx;
+	  {
+	       E Tp, Tn, To, Tv, Tt, Tu;
+	       Tp = FMA(KP1_113340798, Th, KP1_326827896 * Tm);
+	       Tn = FNMS(KP642787609, Tm, KP766044443 * Th);
+	       To = Tc - Tn;
+	       O[WS(os, 1)] = FMA(KP2_000000000, Tn, Tc);
+	       O[WS(os, 7)] = To + Tp;
+	       O[WS(os, 4)] = To - Tp;
+	       Tv = FMA(KP1_705737063, Tr, KP300767466 * Ts);
+	       Tt = FNMS(KP984807753, Ts, KP173648177 * Tr);
+	       Tu = Tq - Tt;
+	       O[WS(os, 2)] = FMA(KP2_000000000, Tt, Tq);
+	       O[WS(os, 8)] = Tu + Tv;
+	       O[WS(os, 5)] = Tu - Tv;
+	  }
+     }
+}
+
+static const khc2r_desc desc = { 9, "hc2r_9", {22, 8, 10, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_hc2r_9) (planner *p) {
+     X(khc2r_register) (p, hc2r_9, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/hcodlist.c b/src/fftw3/rdft/codelets/hc2r/hcodlist.c
new file mode 100644
index 0000000..d5d2f07
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/hcodlist.c
@@ -0,0 +1,102 @@
+#include "ifftw.h"
+
+extern void X(codelet_hc2r_3)(planner *);
+extern void X(codelet_hc2r_4)(planner *);
+extern void X(codelet_hc2r_5)(planner *);
+extern void X(codelet_hc2r_6)(planner *);
+extern void X(codelet_hc2r_7)(planner *);
+extern void X(codelet_hc2r_8)(planner *);
+extern void X(codelet_hc2r_9)(planner *);
+extern void X(codelet_hc2r_10)(planner *);
+extern void X(codelet_hc2r_11)(planner *);
+extern void X(codelet_hc2r_12)(planner *);
+extern void X(codelet_hc2r_13)(planner *);
+extern void X(codelet_hc2r_14)(planner *);
+extern void X(codelet_hc2r_15)(planner *);
+extern void X(codelet_hc2r_16)(planner *);
+extern void X(codelet_hc2r_32)(planner *);
+extern void X(codelet_mhc2r_32)(planner *);
+extern void X(codelet_mhc2r_64)(planner *);
+extern void X(codelet_mhc2r_128)(planner *);
+extern void X(codelet_hb_2)(planner *);
+extern void X(codelet_hb_3)(planner *);
+extern void X(codelet_hb_4)(planner *);
+extern void X(codelet_hb_5)(planner *);
+extern void X(codelet_hb_6)(planner *);
+extern void X(codelet_hb_7)(planner *);
+extern void X(codelet_hb_8)(planner *);
+extern void X(codelet_hb_9)(planner *);
+extern void X(codelet_hb_10)(planner *);
+extern void X(codelet_hb_12)(planner *);
+extern void X(codelet_hb_15)(planner *);
+extern void X(codelet_hb_16)(planner *);
+extern void X(codelet_hb_32)(planner *);
+extern void X(codelet_hb_64)(planner *);
+extern void X(codelet_hc2rIII_2)(planner *);
+extern void X(codelet_hc2rIII_3)(planner *);
+extern void X(codelet_hc2rIII_4)(planner *);
+extern void X(codelet_hc2rIII_5)(planner *);
+extern void X(codelet_hc2rIII_6)(planner *);
+extern void X(codelet_hc2rIII_7)(planner *);
+extern void X(codelet_hc2rIII_8)(planner *);
+extern void X(codelet_hc2rIII_9)(planner *);
+extern void X(codelet_hc2rIII_10)(planner *);
+extern void X(codelet_hc2rIII_12)(planner *);
+extern void X(codelet_hc2rIII_15)(planner *);
+extern void X(codelet_hc2rIII_16)(planner *);
+extern void X(codelet_hc2rIII_32)(planner *);
+extern void X(codelet_mhc2rIII_32)(planner *);
+extern void X(codelet_mhc2rIII_64)(planner *);
+
+
+extern const solvtab X(solvtab_rdft_hc2r);
+const solvtab X(solvtab_rdft_hc2r) = {
+   SOLVTAB(X(codelet_hc2r_3)),
+   SOLVTAB(X(codelet_hc2r_4)),
+   SOLVTAB(X(codelet_hc2r_5)),
+   SOLVTAB(X(codelet_hc2r_6)),
+   SOLVTAB(X(codelet_hc2r_7)),
+   SOLVTAB(X(codelet_hc2r_8)),
+   SOLVTAB(X(codelet_hc2r_9)),
+   SOLVTAB(X(codelet_hc2r_10)),
+   SOLVTAB(X(codelet_hc2r_11)),
+   SOLVTAB(X(codelet_hc2r_12)),
+   SOLVTAB(X(codelet_hc2r_13)),
+   SOLVTAB(X(codelet_hc2r_14)),
+   SOLVTAB(X(codelet_hc2r_15)),
+   SOLVTAB(X(codelet_hc2r_16)),
+   SOLVTAB(X(codelet_hc2r_32)),
+   SOLVTAB(X(codelet_mhc2r_32)),
+   SOLVTAB(X(codelet_mhc2r_64)),
+   SOLVTAB(X(codelet_mhc2r_128)),
+   SOLVTAB(X(codelet_hb_2)),
+   SOLVTAB(X(codelet_hb_3)),
+   SOLVTAB(X(codelet_hb_4)),
+   SOLVTAB(X(codelet_hb_5)),
+   SOLVTAB(X(codelet_hb_6)),
+   SOLVTAB(X(codelet_hb_7)),
+   SOLVTAB(X(codelet_hb_8)),
+   SOLVTAB(X(codelet_hb_9)),
+   SOLVTAB(X(codelet_hb_10)),
+   SOLVTAB(X(codelet_hb_12)),
+   SOLVTAB(X(codelet_hb_15)),
+   SOLVTAB(X(codelet_hb_16)),
+   SOLVTAB(X(codelet_hb_32)),
+   SOLVTAB(X(codelet_hb_64)),
+   SOLVTAB(X(codelet_hc2rIII_2)),
+   SOLVTAB(X(codelet_hc2rIII_3)),
+   SOLVTAB(X(codelet_hc2rIII_4)),
+   SOLVTAB(X(codelet_hc2rIII_5)),
+   SOLVTAB(X(codelet_hc2rIII_6)),
+   SOLVTAB(X(codelet_hc2rIII_7)),
+   SOLVTAB(X(codelet_hc2rIII_8)),
+   SOLVTAB(X(codelet_hc2rIII_9)),
+   SOLVTAB(X(codelet_hc2rIII_10)),
+   SOLVTAB(X(codelet_hc2rIII_12)),
+   SOLVTAB(X(codelet_hc2rIII_15)),
+   SOLVTAB(X(codelet_hc2rIII_16)),
+   SOLVTAB(X(codelet_hc2rIII_32)),
+   SOLVTAB(X(codelet_mhc2rIII_32)),
+   SOLVTAB(X(codelet_mhc2rIII_64)),
+   SOLVTAB_END
+};
diff --git a/src/fftw3/rdft/codelets/hc2r/mhc2rIII_32.c b/src/fftw3/rdft/codelets/hc2r/mhc2rIII_32.c
new file mode 100644
index 0000000..5ae4ebf
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/mhc2rIII_32.c
@@ -0,0 +1,381 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:12:16 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r_noinline -compact -variables 4 -sign 1 -n 32 -name mhc2rIII_32 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 174 FP additions, 84 FP multiplications,
+ * (or, 138 additions, 48 multiplications, 36 fused multiply/add),
+ * 65 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: mhc2rIII_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: mhc2rIII_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: mhc2rIII_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void mhc2rIII_32_0(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os)
+{
+     DK(KP1_913880671, +1.913880671464417729871595773960539938965698411);
+     DK(KP580569354, +0.580569354508924735272384751634790549382952557);
+     DK(KP942793473, +0.942793473651995297112775251810508755314920638);
+     DK(KP1_763842528, +1.763842528696710059425513727320776699016885241);
+     DK(KP1_546020906, +1.546020906725473921621813219516939601942082586);
+     DK(KP1_268786568, +1.268786568327290996430343226450986741351374190);
+     DK(KP196034280, +0.196034280659121203988391127777283691722273346);
+     DK(KP1_990369453, +1.990369453344393772489673906218959843150949737);
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     DK(KP1_961570560, +1.961570560806460898252364472268478073947867462);
+     DK(KP390180644, +0.390180644032256535696569736954044481855383236);
+     DK(KP1_111140466, +1.111140466039204449485661627897065748749874382);
+     DK(KP1_662939224, +1.662939224605090474157576755235811513477121624);
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     {
+	  E T7, T2i, T2F, Tz, T1k, T1I, T1Z, T1x, Te, T22, T2E, T2j, T1f, T1y, TK;
+	  E T1J, Tm, T2B, TW, T1a, T1C, T1L, T28, T2l, Tt, T2A, T17, T1b, T1F, T1M;
+	  E T2d, T2m;
+	  {
+	       E T3, Tv, T1j, T2h, T6, T1g, Ty, T2g;
+	       {
+		    E T1, T2, T1h, T1i;
+		    T1 = ri[0];
+		    T2 = ri[WS(ris, 15)];
+		    T3 = T1 + T2;
+		    Tv = T1 - T2;
+		    T1h = ii[0];
+		    T1i = ii[WS(iis, 15)];
+		    T1j = T1h + T1i;
+		    T2h = T1i - T1h;
+	       }
+	       {
+		    E T4, T5, Tw, Tx;
+		    T4 = ri[WS(ris, 8)];
+		    T5 = ri[WS(ris, 7)];
+		    T6 = T4 + T5;
+		    T1g = T4 - T5;
+		    Tw = ii[WS(iis, 8)];
+		    Tx = ii[WS(iis, 7)];
+		    Ty = Tw + Tx;
+		    T2g = Tw - Tx;
+	       }
+	       T7 = T3 + T6;
+	       T2i = T2g + T2h;
+	       T2F = T2h - T2g;
+	       Tz = Tv - Ty;
+	       T1k = T1g + T1j;
+	       T1I = T1g - T1j;
+	       T1Z = T3 - T6;
+	       T1x = Tv + Ty;
+	  }
+	  {
+	       E Ta, TA, TD, T21, Td, TF, TI, T20;
+	       {
+		    E T8, T9, TB, TC;
+		    T8 = ri[WS(ris, 4)];
+		    T9 = ri[WS(ris, 11)];
+		    Ta = T8 + T9;
+		    TA = T8 - T9;
+		    TB = ii[WS(iis, 4)];
+		    TC = ii[WS(iis, 11)];
+		    TD = TB + TC;
+		    T21 = TB - TC;
+	       }
+	       {
+		    E Tb, Tc, TG, TH;
+		    Tb = ri[WS(ris, 3)];
+		    Tc = ri[WS(ris, 12)];
+		    Td = Tb + Tc;
+		    TF = Tb - Tc;
+		    TG = ii[WS(iis, 3)];
+		    TH = ii[WS(iis, 12)];
+		    TI = TG + TH;
+		    T20 = TH - TG;
+	       }
+	       Te = Ta + Td;
+	       T22 = T20 - T21;
+	       T2E = T21 + T20;
+	       T2j = Ta - Td;
+	       {
+		    E T1d, T1e, TE, TJ;
+		    T1d = TA + TD;
+		    T1e = TF + TI;
+		    T1f = KP707106781 * (T1d - T1e);
+		    T1y = KP707106781 * (T1d + T1e);
+		    TE = TA - TD;
+		    TJ = TF - TI;
+		    TK = KP707106781 * (TE + TJ);
+		    T1J = KP707106781 * (TE - TJ);
+	       }
+	  }
+	  {
+	       E Ti, TM, TU, T25, Tl, TR, TP, T26, TQ, TV;
+	       {
+		    E Tg, Th, TS, TT;
+		    Tg = ri[WS(ris, 2)];
+		    Th = ri[WS(ris, 13)];
+		    Ti = Tg + Th;
+		    TM = Tg - Th;
+		    TS = ii[WS(iis, 2)];
+		    TT = ii[WS(iis, 13)];
+		    TU = TS + TT;
+		    T25 = TS - TT;
+	       }
+	       {
+		    E Tj, Tk, TN, TO;
+		    Tj = ri[WS(ris, 10)];
+		    Tk = ri[WS(ris, 5)];
+		    Tl = Tj + Tk;
+		    TR = Tj - Tk;
+		    TN = ii[WS(iis, 10)];
+		    TO = ii[WS(iis, 5)];
+		    TP = TN + TO;
+		    T26 = TN - TO;
+	       }
+	       Tm = Ti + Tl;
+	       T2B = T26 + T25;
+	       TQ = TM - TP;
+	       TV = TR + TU;
+	       TW = FNMS(KP382683432, TV, KP923879532 * TQ);
+	       T1a = FMA(KP382683432, TQ, KP923879532 * TV);
+	       {
+		    E T1A, T1B, T24, T27;
+		    T1A = TM + TP;
+		    T1B = TU - TR;
+		    T1C = FNMS(KP923879532, T1B, KP382683432 * T1A);
+		    T1L = FMA(KP923879532, T1A, KP382683432 * T1B);
+		    T24 = Ti - Tl;
+		    T27 = T25 - T26;
+		    T28 = T24 - T27;
+		    T2l = T24 + T27;
+	       }
+	  }
+	  {
+	       E Tp, TX, T15, T2a, Ts, T12, T10, T2b, T11, T16;
+	       {
+		    E Tn, To, T13, T14;
+		    Tn = ri[WS(ris, 1)];
+		    To = ri[WS(ris, 14)];
+		    Tp = Tn + To;
+		    TX = Tn - To;
+		    T13 = ii[WS(iis, 1)];
+		    T14 = ii[WS(iis, 14)];
+		    T15 = T13 + T14;
+		    T2a = T14 - T13;
+	       }
+	       {
+		    E Tq, Tr, TY, TZ;
+		    Tq = ri[WS(ris, 6)];
+		    Tr = ri[WS(ris, 9)];
+		    Ts = Tq + Tr;
+		    T12 = Tq - Tr;
+		    TY = ii[WS(iis, 6)];
+		    TZ = ii[WS(iis, 9)];
+		    T10 = TY + TZ;
+		    T2b = TY - TZ;
+	       }
+	       Tt = Tp + Ts;
+	       T2A = T2b + T2a;
+	       T11 = TX - T10;
+	       T16 = T12 - T15;
+	       T17 = FMA(KP923879532, T11, KP382683432 * T16);
+	       T1b = FNMS(KP382683432, T11, KP923879532 * T16);
+	       {
+		    E T1D, T1E, T29, T2c;
+		    T1D = TX + T10;
+		    T1E = T12 + T15;
+		    T1F = FNMS(KP923879532, T1E, KP382683432 * T1D);
+		    T1M = FMA(KP923879532, T1D, KP382683432 * T1E);
+		    T29 = Tp - Ts;
+		    T2c = T2a - T2b;
+		    T2d = T29 + T2c;
+		    T2m = T2c - T29;
+	       }
+	  }
+	  {
+	       E Tf, Tu, T2L, T2M, T2N, T2O;
+	       Tf = T7 + Te;
+	       Tu = Tm + Tt;
+	       T2L = Tf - Tu;
+	       T2M = T2B + T2A;
+	       T2N = T2F - T2E;
+	       T2O = T2M + T2N;
+	       O[0] = KP2_000000000 * (Tf + Tu);
+	       O[WS(os, 16)] = KP2_000000000 * (T2N - T2M);
+	       O[WS(os, 8)] = KP1_414213562 * (T2L + T2O);
+	       O[WS(os, 24)] = KP1_414213562 * (T2O - T2L);
+	  }
+	  {
+	       E T2t, T2x, T2w, T2y;
+	       {
+		    E T2r, T2s, T2u, T2v;
+		    T2r = T1Z - T22;
+		    T2s = KP707106781 * (T2m - T2l);
+		    T2t = T2r + T2s;
+		    T2x = T2r - T2s;
+		    T2u = T2j + T2i;
+		    T2v = KP707106781 * (T28 - T2d);
+		    T2w = T2u - T2v;
+		    T2y = T2v + T2u;
+	       }
+	       O[WS(os, 6)] = FMA(KP1_662939224, T2t, KP1_111140466 * T2w);
+	       O[WS(os, 30)] = FNMS(KP1_961570560, T2x, KP390180644 * T2y);
+	       O[WS(os, 22)] = FNMS(KP1_111140466, T2t, KP1_662939224 * T2w);
+	       O[WS(os, 14)] = FMA(KP390180644, T2x, KP1_961570560 * T2y);
+	  }
+	  {
+	       E T2D, T2J, T2I, T2K;
+	       {
+		    E T2z, T2C, T2G, T2H;
+		    T2z = T7 - Te;
+		    T2C = T2A - T2B;
+		    T2D = T2z + T2C;
+		    T2J = T2z - T2C;
+		    T2G = T2E + T2F;
+		    T2H = Tm - Tt;
+		    T2I = T2G - T2H;
+		    T2K = T2H + T2G;
+	       }
+	       O[WS(os, 4)] = FMA(KP1_847759065, T2D, KP765366864 * T2I);
+	       O[WS(os, 28)] = FNMS(KP1_847759065, T2J, KP765366864 * T2K);
+	       O[WS(os, 20)] = FNMS(KP765366864, T2D, KP1_847759065 * T2I);
+	       O[WS(os, 12)] = FMA(KP765366864, T2J, KP1_847759065 * T2K);
+	  }
+	  {
+	       E T19, T1n, T1m, T1o;
+	       {
+		    E TL, T18, T1c, T1l;
+		    TL = Tz + TK;
+		    T18 = TW + T17;
+		    T19 = TL + T18;
+		    T1n = TL - T18;
+		    T1c = T1a + T1b;
+		    T1l = T1f + T1k;
+		    T1m = T1c + T1l;
+		    T1o = T1c - T1l;
+	       }
+	       O[WS(os, 1)] = FNMS(KP196034280, T1m, KP1_990369453 * T19);
+	       O[WS(os, 25)] = FNMS(KP1_546020906, T1n, KP1_268786568 * T1o);
+	       O[WS(os, 17)] = -(FMA(KP196034280, T19, KP1_990369453 * T1m));
+	       O[WS(os, 9)] = FMA(KP1_268786568, T1n, KP1_546020906 * T1o);
+	  }
+	  {
+	       E T1r, T1v, T1u, T1w;
+	       {
+		    E T1p, T1q, T1s, T1t;
+		    T1p = Tz - TK;
+		    T1q = T1b - T1a;
+		    T1r = T1p + T1q;
+		    T1v = T1p - T1q;
+		    T1s = T1f - T1k;
+		    T1t = TW - T17;
+		    T1u = T1s - T1t;
+		    T1w = T1t + T1s;
+	       }
+	       O[WS(os, 5)] = FMA(KP1_763842528, T1r, KP942793473 * T1u);
+	       O[WS(os, 29)] = FNMS(KP1_913880671, T1v, KP580569354 * T1w);
+	       O[WS(os, 21)] = FNMS(KP942793473, T1r, KP1_763842528 * T1u);
+	       O[WS(os, 13)] = FMA(KP580569354, T1v, KP1_913880671 * T1w);
+	  }
+	  {
+	       E T1T, T1X, T1W, T1Y;
+	       {
+		    E T1R, T1S, T1U, T1V;
+		    T1R = T1x + T1y;
+		    T1S = T1L + T1M;
+		    T1T = T1R - T1S;
+		    T1X = T1R + T1S;
+		    T1U = T1J + T1I;
+		    T1V = T1C - T1F;
+		    T1W = T1U - T1V;
+		    T1Y = T1V + T1U;
+	       }
+	       O[WS(os, 7)] = FMA(KP1_546020906, T1T, KP1_268786568 * T1W);
+	       O[WS(os, 31)] = FNMS(KP1_990369453, T1X, KP196034280 * T1Y);
+	       O[WS(os, 23)] = FNMS(KP1_268786568, T1T, KP1_546020906 * T1W);
+	       O[WS(os, 15)] = FMA(KP196034280, T1X, KP1_990369453 * T1Y);
+	  }
+	  {
+	       E T2f, T2p, T2o, T2q;
+	       {
+		    E T23, T2e, T2k, T2n;
+		    T23 = T1Z + T22;
+		    T2e = KP707106781 * (T28 + T2d);
+		    T2f = T23 + T2e;
+		    T2p = T23 - T2e;
+		    T2k = T2i - T2j;
+		    T2n = KP707106781 * (T2l + T2m);
+		    T2o = T2k - T2n;
+		    T2q = T2n + T2k;
+	       }
+	       O[WS(os, 2)] = FMA(KP1_961570560, T2f, KP390180644 * T2o);
+	       O[WS(os, 26)] = FNMS(KP1_662939224, T2p, KP1_111140466 * T2q);
+	       O[WS(os, 18)] = FNMS(KP390180644, T2f, KP1_961570560 * T2o);
+	       O[WS(os, 10)] = FMA(KP1_111140466, T2p, KP1_662939224 * T2q);
+	  }
+	  {
+	       E T1H, T1P, T1O, T1Q;
+	       {
+		    E T1z, T1G, T1K, T1N;
+		    T1z = T1x - T1y;
+		    T1G = T1C + T1F;
+		    T1H = T1z + T1G;
+		    T1P = T1z - T1G;
+		    T1K = T1I - T1J;
+		    T1N = T1L - T1M;
+		    T1O = T1K - T1N;
+		    T1Q = T1N + T1K;
+	       }
+	       O[WS(os, 3)] = FMA(KP1_913880671, T1H, KP580569354 * T1O);
+	       O[WS(os, 27)] = FNMS(KP1_763842528, T1P, KP942793473 * T1Q);
+	       O[WS(os, 19)] = FNMS(KP580569354, T1H, KP1_913880671 * T1O);
+	       O[WS(os, 11)] = FMA(KP942793473, T1P, KP1_763842528 * T1Q);
+	  }
+     }
+}
+
+static void mhc2rIII_32(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  mhc2rIII_32_0(ri, ii, O, ris, iis, os);
+	  ri += ivs;
+	  ii += ivs;
+	  O += ovs;
+     }
+}
+
+static const khc2r_desc desc = { 32, "mhc2rIII_32", {138, 48, 36, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_mhc2rIII_32) (planner *p) {
+     X(khc2rIII_register) (p, mhc2rIII_32, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/mhc2rIII_64.c b/src/fftw3/rdft/codelets/hc2r/mhc2rIII_64.c
new file mode 100644
index 0000000..42e7790
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/mhc2rIII_64.c
@@ -0,0 +1,819 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:12:24 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r_noinline -compact -variables 4 -sign 1 -n 64 -name mhc2rIII_64 -dft-III -include hc2rIII.h */
+
+/*
+ * This function contains 434 FP additions, 208 FP multiplications,
+ * (or, 342 additions, 116 multiplications, 92 fused multiply/add),
+ * 129 stack variables, and 128 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: mhc2rIII_64.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: mhc2rIII_64.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: mhc2rIII_64.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2rIII.h"
+
+static void mhc2rIII_64_0(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os)
+{
+     DK(KP1_343117909, +1.343117909694036801250753700854843606457501264);
+     DK(KP1_481902250, +1.481902250709918182351233794990325459457910619);
+     DK(KP1_807978586, +1.807978586246886663172400594461074097420264050);
+     DK(KP855110186, +0.855110186860564188641933713777597068609157259);
+     DK(KP1_997590912, +1.997590912410344785429543209518201388886407229);
+     DK(KP098135348, +0.098135348654836028509909953885365316629490726);
+     DK(KP673779706, +0.673779706784440101378506425238295140955533559);
+     DK(KP1_883088130, +1.883088130366041556825018805199004714371179592);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP1_191398608, +1.191398608984866686934073057659939779023852677);
+     DK(KP1_606415062, +1.606415062961289819613353025926283847759138854);
+     DK(KP1_715457220, +1.715457220000544139804539968569540274084981599);
+     DK(KP1_028205488, +1.028205488386443453187387677937631545216098241);
+     DK(KP1_978353019, +1.978353019929561946903347476032486127967379067);
+     DK(KP293460948, +0.293460948910723503317700259293435639412430633);
+     DK(KP485960359, +0.485960359806527779896548324154942236641981567);
+     DK(KP1_940062506, +1.940062506389087985207968414572200502913731924);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP1_268786568, +1.268786568327290996430343226450986741351374190);
+     DK(KP1_546020906, +1.546020906725473921621813219516939601942082586);
+     DK(KP1_763842528, +1.763842528696710059425513727320776699016885241);
+     DK(KP942793473, +0.942793473651995297112775251810508755314920638);
+     DK(KP1_990369453, +1.990369453344393772489673906218959843150949737);
+     DK(KP196034280, +0.196034280659121203988391127777283691722273346);
+     DK(KP580569354, +0.580569354508924735272384751634790549382952557);
+     DK(KP1_913880671, +1.913880671464417729871595773960539938965698411);
+     DK(KP1_662939224, +1.662939224605090474157576755235811513477121624);
+     DK(KP1_111140466, +1.111140466039204449485661627897065748749874382);
+     DK(KP390180644, +0.390180644032256535696569736954044481855383236);
+     DK(KP1_961570560, +1.961570560806460898252364472268478073947867462);
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     {
+	  E T15, T3t, T3U, T2N, Tf, T6b, T6u, T6R, T4L, T5J, T1g, T3V, T5q, T5U, T2I;
+	  E T3u, Tu, T6v, T4V, T5s, T6e, T6Q, T1s, T2D, T1D, T2E, T3B, T3Y, T4Q, T5r;
+	  E T3y, T3X, TK, T6g, T57, T5N, T6j, T6N, T1W, T34, T25, T35, T3J, T4j, T52;
+	  E T5M, T3G, T4i, TZ, T6l, T5i, T5Q, T6o, T6M, T2n, T37, T2w, T38, T3Q, T4m;
+	  E T5d, T5P, T3N, T4l;
+	  {
+	       E T3, T11, T2M, T5n, T6, T2J, T14, T5m, Ta, T16, T19, T4J, Td, T1b, T1e;
+	       E T4I;
+	       {
+		    E T1, T2, T2K, T2L;
+		    T1 = ri[0];
+		    T2 = ri[WS(ris, 31)];
+		    T3 = T1 + T2;
+		    T11 = T1 - T2;
+		    T2K = ii[0];
+		    T2L = ii[WS(iis, 31)];
+		    T2M = T2K + T2L;
+		    T5n = T2L - T2K;
+	       }
+	       {
+		    E T4, T5, T12, T13;
+		    T4 = ri[WS(ris, 16)];
+		    T5 = ri[WS(ris, 15)];
+		    T6 = T4 + T5;
+		    T2J = T4 - T5;
+		    T12 = ii[WS(iis, 16)];
+		    T13 = ii[WS(iis, 15)];
+		    T14 = T12 + T13;
+		    T5m = T12 - T13;
+	       }
+	       {
+		    E T8, T9, T17, T18;
+		    T8 = ri[WS(ris, 8)];
+		    T9 = ri[WS(ris, 23)];
+		    Ta = T8 + T9;
+		    T16 = T8 - T9;
+		    T17 = ii[WS(iis, 8)];
+		    T18 = ii[WS(iis, 23)];
+		    T19 = T17 + T18;
+		    T4J = T17 - T18;
+	       }
+	       {
+		    E Tb, Tc, T1c, T1d;
+		    Tb = ri[WS(ris, 7)];
+		    Tc = ri[WS(ris, 24)];
+		    Td = Tb + Tc;
+		    T1b = Tb - Tc;
+		    T1c = ii[WS(iis, 7)];
+		    T1d = ii[WS(iis, 24)];
+		    T1e = T1c + T1d;
+		    T4I = T1d - T1c;
+	       }
+	       {
+		    E T7, Te, T1a, T1f;
+		    T15 = T11 - T14;
+		    T3t = T11 + T14;
+		    T3U = T2J - T2M;
+		    T2N = T2J + T2M;
+		    T7 = T3 + T6;
+		    Te = Ta + Td;
+		    Tf = T7 + Te;
+		    T6b = T7 - Te;
+		    {
+			 E T6s, T6t, T4H, T4K;
+			 T6s = T4J + T4I;
+			 T6t = T5n - T5m;
+			 T6u = T6s + T6t;
+			 T6R = T6t - T6s;
+			 T4H = T3 - T6;
+			 T4K = T4I - T4J;
+			 T4L = T4H + T4K;
+			 T5J = T4H - T4K;
+		    }
+		    T1a = T16 - T19;
+		    T1f = T1b - T1e;
+		    T1g = KP707106781 * (T1a + T1f);
+		    T3V = KP707106781 * (T1a - T1f);
+		    {
+			 E T5o, T5p, T2G, T2H;
+			 T5o = T5m + T5n;
+			 T5p = Ta - Td;
+			 T5q = T5o - T5p;
+			 T5U = T5p + T5o;
+			 T2G = T16 + T19;
+			 T2H = T1b + T1e;
+			 T2I = KP707106781 * (T2G - T2H);
+			 T3u = KP707106781 * (T2G + T2H);
+		    }
+	       }
+	  }
+	  {
+	       E Ti, T1i, T1q, T4N, Tl, T1n, T1l, T4O, Tp, T1t, T1B, T4S, Ts, T1y, T1w;
+	       E T4T;
+	       {
+		    E Tg, Th, T1o, T1p;
+		    Tg = ri[WS(ris, 4)];
+		    Th = ri[WS(ris, 27)];
+		    Ti = Tg + Th;
+		    T1i = Tg - Th;
+		    T1o = ii[WS(iis, 4)];
+		    T1p = ii[WS(iis, 27)];
+		    T1q = T1o + T1p;
+		    T4N = T1o - T1p;
+	       }
+	       {
+		    E Tj, Tk, T1j, T1k;
+		    Tj = ri[WS(ris, 20)];
+		    Tk = ri[WS(ris, 11)];
+		    Tl = Tj + Tk;
+		    T1n = Tj - Tk;
+		    T1j = ii[WS(iis, 20)];
+		    T1k = ii[WS(iis, 11)];
+		    T1l = T1j + T1k;
+		    T4O = T1j - T1k;
+	       }
+	       {
+		    E Tn, To, T1z, T1A;
+		    Tn = ri[WS(ris, 3)];
+		    To = ri[WS(ris, 28)];
+		    Tp = Tn + To;
+		    T1t = Tn - To;
+		    T1z = ii[WS(iis, 3)];
+		    T1A = ii[WS(iis, 28)];
+		    T1B = T1z + T1A;
+		    T4S = T1A - T1z;
+	       }
+	       {
+		    E Tq, Tr, T1u, T1v;
+		    Tq = ri[WS(ris, 12)];
+		    Tr = ri[WS(ris, 19)];
+		    Ts = Tq + Tr;
+		    T1y = Tq - Tr;
+		    T1u = ii[WS(iis, 12)];
+		    T1v = ii[WS(iis, 19)];
+		    T1w = T1u + T1v;
+		    T4T = T1u - T1v;
+	       }
+	       {
+		    E Tm, Tt, T4R, T4U;
+		    Tm = Ti + Tl;
+		    Tt = Tp + Ts;
+		    Tu = Tm + Tt;
+		    T6v = Tm - Tt;
+		    T4R = Tp - Ts;
+		    T4U = T4S - T4T;
+		    T4V = T4R + T4U;
+		    T5s = T4U - T4R;
+	       }
+	       {
+		    E T6c, T6d, T1m, T1r;
+		    T6c = T4T + T4S;
+		    T6d = T4O + T4N;
+		    T6e = T6c - T6d;
+		    T6Q = T6d + T6c;
+		    T1m = T1i - T1l;
+		    T1r = T1n + T1q;
+		    T1s = FNMS(KP382683432, T1r, KP923879532 * T1m);
+		    T2D = FMA(KP382683432, T1m, KP923879532 * T1r);
+	       }
+	       {
+		    E T1x, T1C, T3z, T3A;
+		    T1x = T1t - T1w;
+		    T1C = T1y - T1B;
+		    T1D = FMA(KP923879532, T1x, KP382683432 * T1C);
+		    T2E = FNMS(KP382683432, T1x, KP923879532 * T1C);
+		    T3z = T1t + T1w;
+		    T3A = T1y + T1B;
+		    T3B = FNMS(KP923879532, T3A, KP382683432 * T3z);
+		    T3Y = FMA(KP923879532, T3z, KP382683432 * T3A);
+	       }
+	       {
+		    E T4M, T4P, T3w, T3x;
+		    T4M = Ti - Tl;
+		    T4P = T4N - T4O;
+		    T4Q = T4M - T4P;
+		    T5r = T4M + T4P;
+		    T3w = T1i + T1l;
+		    T3x = T1q - T1n;
+		    T3y = FNMS(KP923879532, T3x, KP382683432 * T3w);
+		    T3X = FMA(KP923879532, T3w, KP382683432 * T3x);
+	       }
+	  }
+	  {
+	       E Ty, T1G, T23, T54, TB, T20, T1J, T55, TI, T4Z, T1U, T1Y, TF, T50, T1P;
+	       E T1X;
+	       {
+		    E Tw, Tx, T1H, T1I;
+		    Tw = ri[WS(ris, 2)];
+		    Tx = ri[WS(ris, 29)];
+		    Ty = Tw + Tx;
+		    T1G = Tw - Tx;
+		    {
+			 E T21, T22, Tz, TA;
+			 T21 = ii[WS(iis, 2)];
+			 T22 = ii[WS(iis, 29)];
+			 T23 = T21 + T22;
+			 T54 = T21 - T22;
+			 Tz = ri[WS(ris, 18)];
+			 TA = ri[WS(ris, 13)];
+			 TB = Tz + TA;
+			 T20 = Tz - TA;
+		    }
+		    T1H = ii[WS(iis, 18)];
+		    T1I = ii[WS(iis, 13)];
+		    T1J = T1H + T1I;
+		    T55 = T1H - T1I;
+		    {
+			 E TG, TH, T1Q, T1R, T1S, T1T;
+			 TG = ri[WS(ris, 5)];
+			 TH = ri[WS(ris, 26)];
+			 T1Q = TG - TH;
+			 T1R = ii[WS(iis, 5)];
+			 T1S = ii[WS(iis, 26)];
+			 T1T = T1R + T1S;
+			 TI = TG + TH;
+			 T4Z = T1S - T1R;
+			 T1U = T1Q - T1T;
+			 T1Y = T1Q + T1T;
+		    }
+		    {
+			 E TD, TE, T1L, T1M, T1N, T1O;
+			 TD = ri[WS(ris, 10)];
+			 TE = ri[WS(ris, 21)];
+			 T1L = TD - TE;
+			 T1M = ii[WS(iis, 10)];
+			 T1N = ii[WS(iis, 21)];
+			 T1O = T1M + T1N;
+			 TF = TD + TE;
+			 T50 = T1M - T1N;
+			 T1P = T1L - T1O;
+			 T1X = T1L + T1O;
+		    }
+	       }
+	       {
+		    E TC, TJ, T53, T56;
+		    TC = Ty + TB;
+		    TJ = TF + TI;
+		    TK = TC + TJ;
+		    T6g = TC - TJ;
+		    T53 = TF - TI;
+		    T56 = T54 - T55;
+		    T57 = T53 + T56;
+		    T5N = T56 - T53;
+	       }
+	       {
+		    E T6h, T6i, T1K, T1V;
+		    T6h = T55 + T54;
+		    T6i = T50 + T4Z;
+		    T6j = T6h - T6i;
+		    T6N = T6i + T6h;
+		    T1K = T1G - T1J;
+		    T1V = KP707106781 * (T1P + T1U);
+		    T1W = T1K + T1V;
+		    T34 = T1K - T1V;
+	       }
+	       {
+		    E T1Z, T24, T3H, T3I;
+		    T1Z = KP707106781 * (T1X - T1Y);
+		    T24 = T20 + T23;
+		    T25 = T1Z + T24;
+		    T35 = T24 - T1Z;
+		    T3H = KP707106781 * (T1P - T1U);
+		    T3I = T23 - T20;
+		    T3J = T3H + T3I;
+		    T4j = T3I - T3H;
+	       }
+	       {
+		    E T4Y, T51, T3E, T3F;
+		    T4Y = Ty - TB;
+		    T51 = T4Z - T50;
+		    T52 = T4Y + T51;
+		    T5M = T4Y - T51;
+		    T3E = T1G + T1J;
+		    T3F = KP707106781 * (T1X + T1Y);
+		    T3G = T3E - T3F;
+		    T4i = T3E + T3F;
+	       }
+	  }
+	  {
+	       E TN, T27, T2u, T5f, TQ, T2r, T2a, T5g, TX, T5a, T2l, T2p, TU, T5b, T2g;
+	       E T2o;
+	       {
+		    E TL, TM, T28, T29;
+		    TL = ri[WS(ris, 1)];
+		    TM = ri[WS(ris, 30)];
+		    TN = TL + TM;
+		    T27 = TL - TM;
+		    {
+			 E T2s, T2t, TO, TP;
+			 T2s = ii[WS(iis, 1)];
+			 T2t = ii[WS(iis, 30)];
+			 T2u = T2s + T2t;
+			 T5f = T2t - T2s;
+			 TO = ri[WS(ris, 14)];
+			 TP = ri[WS(ris, 17)];
+			 TQ = TO + TP;
+			 T2r = TO - TP;
+		    }
+		    T28 = ii[WS(iis, 14)];
+		    T29 = ii[WS(iis, 17)];
+		    T2a = T28 + T29;
+		    T5g = T28 - T29;
+		    {
+			 E TV, TW, T2h, T2i, T2j, T2k;
+			 TV = ri[WS(ris, 9)];
+			 TW = ri[WS(ris, 22)];
+			 T2h = TV - TW;
+			 T2i = ii[WS(iis, 9)];
+			 T2j = ii[WS(iis, 22)];
+			 T2k = T2i + T2j;
+			 TX = TV + TW;
+			 T5a = T2j - T2i;
+			 T2l = T2h - T2k;
+			 T2p = T2h + T2k;
+		    }
+		    {
+			 E TS, TT, T2c, T2d, T2e, T2f;
+			 TS = ri[WS(ris, 6)];
+			 TT = ri[WS(ris, 25)];
+			 T2c = TS - TT;
+			 T2d = ii[WS(iis, 6)];
+			 T2e = ii[WS(iis, 25)];
+			 T2f = T2d + T2e;
+			 TU = TS + TT;
+			 T5b = T2d - T2e;
+			 T2g = T2c - T2f;
+			 T2o = T2c + T2f;
+		    }
+	       }
+	       {
+		    E TR, TY, T5e, T5h;
+		    TR = TN + TQ;
+		    TY = TU + TX;
+		    TZ = TR + TY;
+		    T6l = TR - TY;
+		    T5e = TU - TX;
+		    T5h = T5f - T5g;
+		    T5i = T5e + T5h;
+		    T5Q = T5h - T5e;
+	       }
+	       {
+		    E T6m, T6n, T2b, T2m;
+		    T6m = T5g + T5f;
+		    T6n = T5b + T5a;
+		    T6o = T6m - T6n;
+		    T6M = T6n + T6m;
+		    T2b = T27 - T2a;
+		    T2m = KP707106781 * (T2g + T2l);
+		    T2n = T2b + T2m;
+		    T37 = T2b - T2m;
+	       }
+	       {
+		    E T2q, T2v, T3O, T3P;
+		    T2q = KP707106781 * (T2o - T2p);
+		    T2v = T2r - T2u;
+		    T2w = T2q + T2v;
+		    T38 = T2v - T2q;
+		    T3O = KP707106781 * (T2g - T2l);
+		    T3P = T2r + T2u;
+		    T3Q = T3O - T3P;
+		    T4m = T3O + T3P;
+	       }
+	       {
+		    E T59, T5c, T3L, T3M;
+		    T59 = TN - TQ;
+		    T5c = T5a - T5b;
+		    T5d = T59 + T5c;
+		    T5P = T59 - T5c;
+		    T3L = T27 + T2a;
+		    T3M = KP707106781 * (T2o + T2p);
+		    T3N = T3L - T3M;
+		    T4l = T3L + T3M;
+	       }
+	  }
+	  {
+	       E Tv, T10, T6X, T6Y, T6Z, T70;
+	       Tv = Tf + Tu;
+	       T10 = TK + TZ;
+	       T6X = Tv - T10;
+	       T6Y = T6N + T6M;
+	       T6Z = T6R - T6Q;
+	       T70 = T6Y + T6Z;
+	       O[0] = KP2_000000000 * (Tv + T10);
+	       O[WS(os, 32)] = KP2_000000000 * (T6Z - T6Y);
+	       O[WS(os, 16)] = KP1_414213562 * (T6X + T70);
+	       O[WS(os, 48)] = KP1_414213562 * (T70 - T6X);
+	  }
+	  {
+	       E T6P, T6V, T6U, T6W;
+	       {
+		    E T6L, T6O, T6S, T6T;
+		    T6L = Tf - Tu;
+		    T6O = T6M - T6N;
+		    T6P = T6L + T6O;
+		    T6V = T6L - T6O;
+		    T6S = T6Q + T6R;
+		    T6T = TK - TZ;
+		    T6U = T6S - T6T;
+		    T6W = T6T + T6S;
+	       }
+	       O[WS(os, 8)] = FMA(KP1_847759065, T6P, KP765366864 * T6U);
+	       O[WS(os, 56)] = FNMS(KP1_847759065, T6V, KP765366864 * T6W);
+	       O[WS(os, 40)] = FNMS(KP765366864, T6P, KP1_847759065 * T6U);
+	       O[WS(os, 24)] = FMA(KP765366864, T6V, KP1_847759065 * T6W);
+	  }
+	  {
+	       E T6f, T6w, T6G, T6D, T6z, T6E, T6q, T6H;
+	       T6f = T6b + T6e;
+	       T6w = T6u - T6v;
+	       T6G = T6v + T6u;
+	       T6D = T6b - T6e;
+	       {
+		    E T6x, T6y, T6k, T6p;
+		    T6x = T6g + T6j;
+		    T6y = T6o - T6l;
+		    T6z = KP707106781 * (T6x + T6y);
+		    T6E = KP707106781 * (T6y - T6x);
+		    T6k = T6g - T6j;
+		    T6p = T6l + T6o;
+		    T6q = KP707106781 * (T6k + T6p);
+		    T6H = KP707106781 * (T6k - T6p);
+	       }
+	       {
+		    E T6r, T6A, T6J, T6K;
+		    T6r = T6f + T6q;
+		    T6A = T6w - T6z;
+		    O[WS(os, 4)] = FMA(KP1_961570560, T6r, KP390180644 * T6A);
+		    O[WS(os, 36)] = FNMS(KP390180644, T6r, KP1_961570560 * T6A);
+		    T6J = T6D - T6E;
+		    T6K = T6H + T6G;
+		    O[WS(os, 28)] = FMA(KP390180644, T6J, KP1_961570560 * T6K);
+		    O[WS(os, 60)] = FNMS(KP1_961570560, T6J, KP390180644 * T6K);
+	       }
+	       {
+		    E T6B, T6C, T6F, T6I;
+		    T6B = T6f - T6q;
+		    T6C = T6z + T6w;
+		    O[WS(os, 20)] = FMA(KP1_111140466, T6B, KP1_662939224 * T6C);
+		    O[WS(os, 52)] = FNMS(KP1_662939224, T6B, KP1_111140466 * T6C);
+		    T6F = T6D + T6E;
+		    T6I = T6G - T6H;
+		    O[WS(os, 12)] = FMA(KP1_662939224, T6F, KP1_111140466 * T6I);
+		    O[WS(os, 44)] = FNMS(KP1_111140466, T6F, KP1_662939224 * T6I);
+	       }
+	  }
+	  {
+	       E T5L, T63, T5W, T66, T5S, T67, T5Z, T64, T5K, T5V;
+	       T5K = KP707106781 * (T5s - T5r);
+	       T5L = T5J + T5K;
+	       T63 = T5J - T5K;
+	       T5V = KP707106781 * (T4Q - T4V);
+	       T5W = T5U - T5V;
+	       T66 = T5V + T5U;
+	       {
+		    E T5O, T5R, T5X, T5Y;
+		    T5O = FNMS(KP923879532, T5N, KP382683432 * T5M);
+		    T5R = FMA(KP382683432, T5P, KP923879532 * T5Q);
+		    T5S = T5O + T5R;
+		    T67 = T5O - T5R;
+		    T5X = FMA(KP923879532, T5M, KP382683432 * T5N);
+		    T5Y = FNMS(KP923879532, T5P, KP382683432 * T5Q);
+		    T5Z = T5X + T5Y;
+		    T64 = T5Y - T5X;
+	       }
+	       {
+		    E T5T, T60, T69, T6a;
+		    T5T = T5L + T5S;
+		    T60 = T5W - T5Z;
+		    O[WS(os, 6)] = FMA(KP1_913880671, T5T, KP580569354 * T60);
+		    O[WS(os, 38)] = FNMS(KP580569354, T5T, KP1_913880671 * T60);
+		    T69 = T63 - T64;
+		    T6a = T67 + T66;
+		    O[WS(os, 30)] = FMA(KP196034280, T69, KP1_990369453 * T6a);
+		    O[WS(os, 62)] = FNMS(KP1_990369453, T69, KP196034280 * T6a);
+	       }
+	       {
+		    E T61, T62, T65, T68;
+		    T61 = T5L - T5S;
+		    T62 = T5Z + T5W;
+		    O[WS(os, 22)] = FMA(KP942793473, T61, KP1_763842528 * T62);
+		    O[WS(os, 54)] = FNMS(KP1_763842528, T61, KP942793473 * T62);
+		    T65 = T63 + T64;
+		    T68 = T66 - T67;
+		    O[WS(os, 14)] = FMA(KP1_546020906, T65, KP1_268786568 * T68);
+		    O[WS(os, 46)] = FNMS(KP1_268786568, T65, KP1_546020906 * T68);
+	       }
+	  }
+	  {
+	       E T4X, T5B, T5u, T5E, T5k, T5F, T5x, T5C, T4W, T5t;
+	       T4W = KP707106781 * (T4Q + T4V);
+	       T4X = T4L + T4W;
+	       T5B = T4L - T4W;
+	       T5t = KP707106781 * (T5r + T5s);
+	       T5u = T5q - T5t;
+	       T5E = T5t + T5q;
+	       {
+		    E T58, T5j, T5v, T5w;
+		    T58 = FNMS(KP382683432, T57, KP923879532 * T52);
+		    T5j = FMA(KP923879532, T5d, KP382683432 * T5i);
+		    T5k = T58 + T5j;
+		    T5F = T58 - T5j;
+		    T5v = FMA(KP382683432, T52, KP923879532 * T57);
+		    T5w = FNMS(KP382683432, T5d, KP923879532 * T5i);
+		    T5x = T5v + T5w;
+		    T5C = T5w - T5v;
+	       }
+	       {
+		    E T5l, T5y, T5H, T5I;
+		    T5l = T4X + T5k;
+		    T5y = T5u - T5x;
+		    O[WS(os, 2)] = FMA(KP1_990369453, T5l, KP196034280 * T5y);
+		    O[WS(os, 34)] = FNMS(KP196034280, T5l, KP1_990369453 * T5y);
+		    T5H = T5B - T5C;
+		    T5I = T5F + T5E;
+		    O[WS(os, 26)] = FMA(KP580569354, T5H, KP1_913880671 * T5I);
+		    O[WS(os, 58)] = FNMS(KP1_913880671, T5H, KP580569354 * T5I);
+	       }
+	       {
+		    E T5z, T5A, T5D, T5G;
+		    T5z = T4X - T5k;
+		    T5A = T5x + T5u;
+		    O[WS(os, 18)] = FMA(KP1_268786568, T5z, KP1_546020906 * T5A);
+		    O[WS(os, 50)] = FNMS(KP1_546020906, T5z, KP1_268786568 * T5A);
+		    T5D = T5B + T5C;
+		    T5G = T5E - T5F;
+		    O[WS(os, 10)] = FMA(KP1_763842528, T5D, KP942793473 * T5G);
+		    O[WS(os, 42)] = FNMS(KP942793473, T5D, KP1_763842528 * T5G);
+	       }
+	  }
+	  {
+	       E T33, T3l, T3h, T3m, T3a, T3p, T3e, T3o;
+	       {
+		    E T31, T32, T3f, T3g;
+		    T31 = T15 - T1g;
+		    T32 = T2E - T2D;
+		    T33 = T31 + T32;
+		    T3l = T31 - T32;
+		    T3f = FMA(KP831469612, T34, KP555570233 * T35);
+		    T3g = FNMS(KP831469612, T37, KP555570233 * T38);
+		    T3h = T3f + T3g;
+		    T3m = T3g - T3f;
+	       }
+	       {
+		    E T36, T39, T3c, T3d;
+		    T36 = FNMS(KP831469612, T35, KP555570233 * T34);
+		    T39 = FMA(KP555570233, T37, KP831469612 * T38);
+		    T3a = T36 + T39;
+		    T3p = T36 - T39;
+		    T3c = T2I - T2N;
+		    T3d = T1s - T1D;
+		    T3e = T3c - T3d;
+		    T3o = T3d + T3c;
+	       }
+	       {
+		    E T3b, T3i, T3r, T3s;
+		    T3b = T33 + T3a;
+		    T3i = T3e - T3h;
+		    O[WS(os, 5)] = FMA(KP1_940062506, T3b, KP485960359 * T3i);
+		    O[WS(os, 37)] = FNMS(KP485960359, T3b, KP1_940062506 * T3i);
+		    T3r = T3l - T3m;
+		    T3s = T3p + T3o;
+		    O[WS(os, 29)] = FMA(KP293460948, T3r, KP1_978353019 * T3s);
+		    O[WS(os, 61)] = FNMS(KP1_978353019, T3r, KP293460948 * T3s);
+	       }
+	       {
+		    E T3j, T3k, T3n, T3q;
+		    T3j = T33 - T3a;
+		    T3k = T3h + T3e;
+		    O[WS(os, 21)] = FMA(KP1_028205488, T3j, KP1_715457220 * T3k);
+		    O[WS(os, 53)] = FNMS(KP1_715457220, T3j, KP1_028205488 * T3k);
+		    T3n = T3l + T3m;
+		    T3q = T3o - T3p;
+		    O[WS(os, 13)] = FMA(KP1_606415062, T3n, KP1_191398608 * T3q);
+		    O[WS(os, 45)] = FNMS(KP1_191398608, T3n, KP1_606415062 * T3q);
+	       }
+	  }
+	  {
+	       E T4h, T4z, T4v, T4A, T4o, T4D, T4s, T4C;
+	       {
+		    E T4f, T4g, T4t, T4u;
+		    T4f = T3t + T3u;
+		    T4g = T3X + T3Y;
+		    T4h = T4f - T4g;
+		    T4z = T4f + T4g;
+		    T4t = FMA(KP980785280, T4i, KP195090322 * T4j);
+		    T4u = FMA(KP980785280, T4l, KP195090322 * T4m);
+		    T4v = T4t - T4u;
+		    T4A = T4t + T4u;
+	       }
+	       {
+		    E T4k, T4n, T4q, T4r;
+		    T4k = FNMS(KP980785280, T4j, KP195090322 * T4i);
+		    T4n = FNMS(KP980785280, T4m, KP195090322 * T4l);
+		    T4o = T4k + T4n;
+		    T4D = T4k - T4n;
+		    T4q = T3V + T3U;
+		    T4r = T3y - T3B;
+		    T4s = T4q - T4r;
+		    T4C = T4r + T4q;
+	       }
+	       {
+		    E T4p, T4w, T4F, T4G;
+		    T4p = T4h + T4o;
+		    T4w = T4s - T4v;
+		    O[WS(os, 7)] = FMA(KP1_883088130, T4p, KP673779706 * T4w);
+		    O[WS(os, 39)] = FNMS(KP673779706, T4p, KP1_883088130 * T4w);
+		    T4F = T4z + T4A;
+		    T4G = T4D + T4C;
+		    O[WS(os, 31)] = FMA(KP098135348, T4F, KP1_997590912 * T4G);
+		    O[WS(os, 63)] = FNMS(KP1_997590912, T4F, KP098135348 * T4G);
+	       }
+	       {
+		    E T4x, T4y, T4B, T4E;
+		    T4x = T4h - T4o;
+		    T4y = T4v + T4s;
+		    O[WS(os, 23)] = FMA(KP855110186, T4x, KP1_807978586 * T4y);
+		    O[WS(os, 55)] = FNMS(KP1_807978586, T4x, KP855110186 * T4y);
+		    T4B = T4z - T4A;
+		    T4E = T4C - T4D;
+		    O[WS(os, 15)] = FMA(KP1_481902250, T4B, KP1_343117909 * T4E);
+		    O[WS(os, 47)] = FNMS(KP1_343117909, T4B, KP1_481902250 * T4E);
+	       }
+	  }
+	  {
+	       E T1F, T2T, T2P, T2W, T2y, T2X, T2C, T2U;
+	       {
+		    E T1h, T1E, T2F, T2O;
+		    T1h = T15 + T1g;
+		    T1E = T1s + T1D;
+		    T1F = T1h + T1E;
+		    T2T = T1h - T1E;
+		    T2F = T2D + T2E;
+		    T2O = T2I + T2N;
+		    T2P = T2F + T2O;
+		    T2W = T2F - T2O;
+	       }
+	       {
+		    E T26, T2x, T2A, T2B;
+		    T26 = FNMS(KP195090322, T25, KP980785280 * T1W);
+		    T2x = FMA(KP980785280, T2n, KP195090322 * T2w);
+		    T2y = T26 + T2x;
+		    T2X = T26 - T2x;
+		    T2A = FMA(KP195090322, T1W, KP980785280 * T25);
+		    T2B = FNMS(KP195090322, T2n, KP980785280 * T2w);
+		    T2C = T2A + T2B;
+		    T2U = T2B - T2A;
+	       }
+	       {
+		    E T2z, T2Q, T2Z, T30;
+		    T2z = T1F + T2y;
+		    T2Q = T2C + T2P;
+		    O[WS(os, 1)] = FNMS(KP098135348, T2Q, KP1_997590912 * T2z);
+		    O[WS(os, 33)] = -(FMA(KP098135348, T2z, KP1_997590912 * T2Q));
+		    T2Z = T2T - T2U;
+		    T30 = T2X + T2W;
+		    O[WS(os, 25)] = FMA(KP673779706, T2Z, KP1_883088130 * T30);
+		    O[WS(os, 57)] = FNMS(KP1_883088130, T2Z, KP673779706 * T30);
+	       }
+	       {
+		    E T2R, T2S, T2V, T2Y;
+		    T2R = T1F - T2y;
+		    T2S = T2C - T2P;
+		    O[WS(os, 17)] = FMA(KP1_343117909, T2R, KP1_481902250 * T2S);
+		    O[WS(os, 49)] = FNMS(KP1_481902250, T2R, KP1_343117909 * T2S);
+		    T2V = T2T + T2U;
+		    T2Y = T2W - T2X;
+		    O[WS(os, 9)] = FMA(KP1_807978586, T2V, KP855110186 * T2Y);
+		    O[WS(os, 41)] = FNMS(KP855110186, T2V, KP1_807978586 * T2Y);
+	       }
+	  }
+	  {
+	       E T3D, T47, T43, T48, T3S, T4b, T40, T4a;
+	       {
+		    E T3v, T3C, T41, T42;
+		    T3v = T3t - T3u;
+		    T3C = T3y + T3B;
+		    T3D = T3v + T3C;
+		    T47 = T3v - T3C;
+		    T41 = FMA(KP555570233, T3G, KP831469612 * T3J);
+		    T42 = FNMS(KP555570233, T3N, KP831469612 * T3Q);
+		    T43 = T41 + T42;
+		    T48 = T42 - T41;
+	       }
+	       {
+		    E T3K, T3R, T3W, T3Z;
+		    T3K = FNMS(KP555570233, T3J, KP831469612 * T3G);
+		    T3R = FMA(KP831469612, T3N, KP555570233 * T3Q);
+		    T3S = T3K + T3R;
+		    T4b = T3K - T3R;
+		    T3W = T3U - T3V;
+		    T3Z = T3X - T3Y;
+		    T40 = T3W - T3Z;
+		    T4a = T3Z + T3W;
+	       }
+	       {
+		    E T3T, T44, T4d, T4e;
+		    T3T = T3D + T3S;
+		    T44 = T40 - T43;
+		    O[WS(os, 3)] = FMA(KP1_978353019, T3T, KP293460948 * T44);
+		    O[WS(os, 35)] = FNMS(KP293460948, T3T, KP1_978353019 * T44);
+		    T4d = T47 - T48;
+		    T4e = T4b + T4a;
+		    O[WS(os, 27)] = FMA(KP485960359, T4d, KP1_940062506 * T4e);
+		    O[WS(os, 59)] = FNMS(KP1_940062506, T4d, KP485960359 * T4e);
+	       }
+	       {
+		    E T45, T46, T49, T4c;
+		    T45 = T3D - T3S;
+		    T46 = T43 + T40;
+		    O[WS(os, 19)] = FMA(KP1_191398608, T45, KP1_606415062 * T46);
+		    O[WS(os, 51)] = FNMS(KP1_606415062, T45, KP1_191398608 * T46);
+		    T49 = T47 + T48;
+		    T4c = T4a - T4b;
+		    O[WS(os, 11)] = FMA(KP1_715457220, T49, KP1_028205488 * T4c);
+		    O[WS(os, 43)] = FNMS(KP1_028205488, T49, KP1_715457220 * T4c);
+	       }
+	  }
+     }
+}
+
+static void mhc2rIII_64(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  mhc2rIII_64_0(ri, ii, O, ris, iis, os);
+	  ri += ivs;
+	  ii += ivs;
+	  O += ovs;
+     }
+}
+
+static const khc2r_desc desc = { 64, "mhc2rIII_64", {342, 116, 92, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_mhc2rIII_64) (planner *p) {
+     X(khc2rIII_register) (p, mhc2rIII_64, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/mhc2r_128.c b/src/fftw3/rdft/codelets/hc2r/mhc2r_128.c
new file mode 100644
index 0000000..27209a3
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/mhc2r_128.c
@@ -0,0 +1,1652 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:20 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r_noinline -compact -variables 4 -sign 1 -n 128 -name mhc2r_128 -include hc2r.h */
+
+/*
+ * This function contains 956 FP additions, 342 FP multiplications,
+ * (or, 812 additions, 198 multiplications, 144 fused multiply/add),
+ * 197 stack variables, and 256 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: mhc2r_128.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: mhc2r_128.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: mhc2r_128.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void mhc2r_128_0(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os)
+{
+     DK(KP1_028205488, +1.028205488386443453187387677937631545216098241);
+     DK(KP1_715457220, +1.715457220000544139804539968569540274084981599);
+     DK(KP1_606415062, +1.606415062961289819613353025926283847759138854);
+     DK(KP1_191398608, +1.191398608984866686934073057659939779023852677);
+     DK(KP1_940062506, +1.940062506389087985207968414572200502913731924);
+     DK(KP485960359, +0.485960359806527779896548324154942236641981567);
+     DK(KP293460948, +0.293460948910723503317700259293435639412430633);
+     DK(KP1_978353019, +1.978353019929561946903347476032486127967379067);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP855110186, +0.855110186860564188641933713777597068609157259);
+     DK(KP1_807978586, +1.807978586246886663172400594461074097420264050);
+     DK(KP1_481902250, +1.481902250709918182351233794990325459457910619);
+     DK(KP1_343117909, +1.343117909694036801250753700854843606457501264);
+     DK(KP1_883088130, +1.883088130366041556825018805199004714371179592);
+     DK(KP673779706, +0.673779706784440101378506425238295140955533559);
+     DK(KP098135348, +0.098135348654836028509909953885365316629490726);
+     DK(KP1_997590912, +1.997590912410344785429543209518201388886407229);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP580569354, +0.580569354508924735272384751634790549382952557);
+     DK(KP1_913880671, +1.913880671464417729871595773960539938965698411);
+     DK(KP942793473, +0.942793473651995297112775251810508755314920638);
+     DK(KP1_763842528, +1.763842528696710059425513727320776699016885241);
+     DK(KP1_111140466, +1.111140466039204449485661627897065748749874382);
+     DK(KP1_662939224, +1.662939224605090474157576755235811513477121624);
+     DK(KP1_268786568, +1.268786568327290996430343226450986741351374190);
+     DK(KP1_546020906, +1.546020906725473921621813219516939601942082586);
+     DK(KP196034280, +0.196034280659121203988391127777283691722273346);
+     DK(KP1_990369453, +1.990369453344393772489673906218959843150949737);
+     DK(KP390180644, +0.390180644032256535696569736954044481855383236);
+     DK(KP1_961570560, +1.961570560806460898252364472268478073947867462);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     {
+	  E Ta, T6q, T2a, T5k, T8x, Tbx, TcF, Ten, Th, T6r, T2j, T5l, T8E, Tby, TcI;
+	  E Teo, Tx, T6t, TcM, Teq, TcP, Ter, T2t, T5n, T2C, T5o, T8Q, TbA, T8X, TbB;
+	  E T6w, T7L, T1j, T6L, Tde, TeC, TdL, TeR, T3v, T5z, T4I, T5O, T9O, TbM, TaV;
+	  E Tc1, T78, T7Z, TN, T6z, TcU, Teu, Td8, Tey, T2N, T5r, T3j, T5v, T9a, TbE;
+	  E T9A, TbI, T6H, T7O, T1O, T7V, T48, T4u, Tds, TeG, T5E, T5K, Taf, TbP, Tdp;
+	  E TeF, T6U, T72, Tam, TbQ, T23, T7U, T4r, T4v, Tdz, TeJ, T5H, T5L, Tay, TbS;
+	  E Tdw, TeI, T6Z, T73, TaF, TbT, T1y, T75, Tdl, TeQ, TdI, TeD, T3O, T5N, T4z;
+	  E T5A, Ta3, Tc0, TaO, TbN, T6O, T80, T12, T6E, Td1, Tex, Td5, Tev, T36, T5u;
+	  E T3a, T5s, T9p, TbH, T9t, TbF, T6C, T7P;
+	  {
+	       E T5, T8s, T3, T8q, T9, T8u, T29, T8v, T6, T26;
+	       {
+		    E T4, T8r, T1, T2;
+		    T4 = ri[WS(ris, 32)];
+		    T5 = KP2_000000000 * T4;
+		    T8r = ii[WS(iis, 32)];
+		    T8s = KP2_000000000 * T8r;
+		    T1 = ri[0];
+		    T2 = ri[WS(ris, 64)];
+		    T3 = T1 + T2;
+		    T8q = T1 - T2;
+		    {
+			 E T7, T8, T27, T28;
+			 T7 = ri[WS(ris, 16)];
+			 T8 = ri[WS(ris, 48)];
+			 T9 = KP2_000000000 * (T7 + T8);
+			 T8u = T7 - T8;
+			 T27 = ii[WS(iis, 16)];
+			 T28 = ii[WS(iis, 48)];
+			 T29 = KP2_000000000 * (T27 - T28);
+			 T8v = T27 + T28;
+		    }
+	       }
+	       T6 = T3 + T5;
+	       Ta = T6 + T9;
+	       T6q = T6 - T9;
+	       T26 = T3 - T5;
+	       T2a = T26 - T29;
+	       T5k = T26 + T29;
+	       {
+		    E T8t, T8w, TcD, TcE;
+		    T8t = T8q - T8s;
+		    T8w = KP1_414213562 * (T8u - T8v);
+		    T8x = T8t + T8w;
+		    Tbx = T8t - T8w;
+		    TcD = T8q + T8s;
+		    TcE = KP1_414213562 * (T8u + T8v);
+		    TcF = TcD - TcE;
+		    Ten = TcD + TcE;
+	       }
+	  }
+	  {
+	       E Td, T8y, T2e, T8C, Tg, T8B, T2h, T8z, T2b, T2i;
+	       {
+		    E Tb, Tc, T2c, T2d;
+		    Tb = ri[WS(ris, 8)];
+		    Tc = ri[WS(ris, 56)];
+		    Td = Tb + Tc;
+		    T8y = Tb - Tc;
+		    T2c = ii[WS(iis, 8)];
+		    T2d = ii[WS(iis, 56)];
+		    T2e = T2c - T2d;
+		    T8C = T2c + T2d;
+	       }
+	       {
+		    E Te, Tf, T2f, T2g;
+		    Te = ri[WS(ris, 40)];
+		    Tf = ri[WS(ris, 24)];
+		    Tg = Te + Tf;
+		    T8B = Te - Tf;
+		    T2f = ii[WS(iis, 40)];
+		    T2g = ii[WS(iis, 24)];
+		    T2h = T2f - T2g;
+		    T8z = T2f + T2g;
+	       }
+	       Th = KP2_000000000 * (Td + Tg);
+	       T6r = KP2_000000000 * (T2h + T2e);
+	       T2b = Td - Tg;
+	       T2i = T2e - T2h;
+	       T2j = KP1_414213562 * (T2b - T2i);
+	       T5l = KP1_414213562 * (T2b + T2i);
+	       {
+		    E T8A, T8D, TcG, TcH;
+		    T8A = T8y - T8z;
+		    T8D = T8B + T8C;
+		    T8E = FNMS(KP765366864, T8D, KP1_847759065 * T8A);
+		    Tby = FMA(KP765366864, T8A, KP1_847759065 * T8D);
+		    TcG = T8y + T8z;
+		    TcH = T8C - T8B;
+		    TcI = FNMS(KP1_847759065, TcH, KP765366864 * TcG);
+		    Teo = FMA(KP1_847759065, TcG, KP765366864 * TcH);
+	       }
+	  }
+	  {
+	       E Tl, T8G, T2x, T8V, To, T8U, T2A, T8H, Tv, T8S, T2o, T8O, Ts, T8R, T2r;
+	       E T8L;
+	       {
+		    E Tj, Tk, T2y, T2z;
+		    Tj = ri[WS(ris, 4)];
+		    Tk = ri[WS(ris, 60)];
+		    Tl = Tj + Tk;
+		    T8G = Tj - Tk;
+		    {
+			 E T2v, T2w, Tm, Tn;
+			 T2v = ii[WS(iis, 4)];
+			 T2w = ii[WS(iis, 60)];
+			 T2x = T2v - T2w;
+			 T8V = T2v + T2w;
+			 Tm = ri[WS(ris, 36)];
+			 Tn = ri[WS(ris, 28)];
+			 To = Tm + Tn;
+			 T8U = Tm - Tn;
+		    }
+		    T2y = ii[WS(iis, 36)];
+		    T2z = ii[WS(iis, 28)];
+		    T2A = T2y - T2z;
+		    T8H = T2y + T2z;
+		    {
+			 E Tt, Tu, T8M, T2m, T2n, T8N;
+			 Tt = ri[WS(ris, 12)];
+			 Tu = ri[WS(ris, 52)];
+			 T8M = Tt - Tu;
+			 T2m = ii[WS(iis, 52)];
+			 T2n = ii[WS(iis, 12)];
+			 T8N = T2n + T2m;
+			 Tv = Tt + Tu;
+			 T8S = T8M + T8N;
+			 T2o = T2m - T2n;
+			 T8O = T8M - T8N;
+		    }
+		    {
+			 E Tq, Tr, T8J, T2p, T2q, T8K;
+			 Tq = ri[WS(ris, 20)];
+			 Tr = ri[WS(ris, 44)];
+			 T8J = Tq - Tr;
+			 T2p = ii[WS(iis, 20)];
+			 T2q = ii[WS(iis, 44)];
+			 T8K = T2p + T2q;
+			 Ts = Tq + Tr;
+			 T8R = T8J + T8K;
+			 T2r = T2p - T2q;
+			 T8L = T8J - T8K;
+		    }
+	       }
+	       {
+		    E Tp, Tw, TcK, TcL;
+		    Tp = Tl + To;
+		    Tw = Ts + Tv;
+		    Tx = KP2_000000000 * (Tp + Tw);
+		    T6t = Tp - Tw;
+		    TcK = T8G + T8H;
+		    TcL = KP707106781 * (T8R + T8S);
+		    TcM = TcK - TcL;
+		    Teq = TcK + TcL;
+	       }
+	       {
+		    E TcN, TcO, T2l, T2s;
+		    TcN = KP707106781 * (T8L - T8O);
+		    TcO = T8V - T8U;
+		    TcP = TcN + TcO;
+		    Ter = TcO - TcN;
+		    T2l = Tl - To;
+		    T2s = T2o - T2r;
+		    T2t = T2l + T2s;
+		    T5n = T2l - T2s;
+	       }
+	       {
+		    E T2u, T2B, T8I, T8P;
+		    T2u = Ts - Tv;
+		    T2B = T2x - T2A;
+		    T2C = T2u + T2B;
+		    T5o = T2B - T2u;
+		    T8I = T8G - T8H;
+		    T8P = KP707106781 * (T8L + T8O);
+		    T8Q = T8I + T8P;
+		    TbA = T8I - T8P;
+	       }
+	       {
+		    E T8T, T8W, T6u, T6v;
+		    T8T = KP707106781 * (T8R - T8S);
+		    T8W = T8U + T8V;
+		    T8X = T8T + T8W;
+		    TbB = T8W - T8T;
+		    T6u = T2A + T2x;
+		    T6v = T2r + T2o;
+		    T6w = T6u - T6v;
+		    T7L = KP2_000000000 * (T6v + T6u);
+	       }
+	  }
+	  {
+	       E T17, T9E, T4D, TaT, T1a, TaS, T4G, T9F, T1h, TaQ, T3q, T9M, T1e, TaP, T3t;
+	       E T9J;
+	       {
+		    E T15, T16, T4E, T4F;
+		    T15 = ri[WS(ris, 1)];
+		    T16 = ri[WS(ris, 63)];
+		    T17 = T15 + T16;
+		    T9E = T15 - T16;
+		    {
+			 E T4B, T4C, T18, T19;
+			 T4B = ii[WS(iis, 1)];
+			 T4C = ii[WS(iis, 63)];
+			 T4D = T4B - T4C;
+			 TaT = T4B + T4C;
+			 T18 = ri[WS(ris, 33)];
+			 T19 = ri[WS(ris, 31)];
+			 T1a = T18 + T19;
+			 TaS = T18 - T19;
+		    }
+		    T4E = ii[WS(iis, 33)];
+		    T4F = ii[WS(iis, 31)];
+		    T4G = T4E - T4F;
+		    T9F = T4E + T4F;
+		    {
+			 E T1f, T1g, T9K, T3o, T3p, T9L;
+			 T1f = ri[WS(ris, 15)];
+			 T1g = ri[WS(ris, 49)];
+			 T9K = T1f - T1g;
+			 T3o = ii[WS(iis, 49)];
+			 T3p = ii[WS(iis, 15)];
+			 T9L = T3p + T3o;
+			 T1h = T1f + T1g;
+			 TaQ = T9K + T9L;
+			 T3q = T3o - T3p;
+			 T9M = T9K - T9L;
+		    }
+		    {
+			 E T1c, T1d, T9H, T3r, T3s, T9I;
+			 T1c = ri[WS(ris, 17)];
+			 T1d = ri[WS(ris, 47)];
+			 T9H = T1c - T1d;
+			 T3r = ii[WS(iis, 17)];
+			 T3s = ii[WS(iis, 47)];
+			 T9I = T3r + T3s;
+			 T1e = T1c + T1d;
+			 TaP = T9H + T9I;
+			 T3t = T3r - T3s;
+			 T9J = T9H - T9I;
+		    }
+	       }
+	       {
+		    E T1b, T1i, Tdc, Tdd;
+		    T1b = T17 + T1a;
+		    T1i = T1e + T1h;
+		    T1j = T1b + T1i;
+		    T6L = T1b - T1i;
+		    Tdc = T9E + T9F;
+		    Tdd = KP707106781 * (TaP + TaQ);
+		    Tde = Tdc - Tdd;
+		    TeC = Tdc + Tdd;
+	       }
+	       {
+		    E TdJ, TdK, T3n, T3u;
+		    TdJ = KP707106781 * (T9J - T9M);
+		    TdK = TaT - TaS;
+		    TdL = TdJ + TdK;
+		    TeR = TdK - TdJ;
+		    T3n = T17 - T1a;
+		    T3u = T3q - T3t;
+		    T3v = T3n + T3u;
+		    T5z = T3n - T3u;
+	       }
+	       {
+		    E T4A, T4H, T9G, T9N;
+		    T4A = T1e - T1h;
+		    T4H = T4D - T4G;
+		    T4I = T4A + T4H;
+		    T5O = T4H - T4A;
+		    T9G = T9E - T9F;
+		    T9N = KP707106781 * (T9J + T9M);
+		    T9O = T9G + T9N;
+		    TbM = T9G - T9N;
+	       }
+	       {
+		    E TaR, TaU, T76, T77;
+		    TaR = KP707106781 * (TaP - TaQ);
+		    TaU = TaS + TaT;
+		    TaV = TaR + TaU;
+		    Tc1 = TaU - TaR;
+		    T76 = T4G + T4D;
+		    T77 = T3t + T3q;
+		    T78 = T76 - T77;
+		    T7Z = T77 + T76;
+	       }
+	  }
+	  {
+	       E TB, T90, T3e, T9y, TE, T9x, T3h, T91, TL, T9v, T2I, T98, TI, T9u, T2L;
+	       E T95;
+	       {
+		    E Tz, TA, T3f, T3g;
+		    Tz = ri[WS(ris, 2)];
+		    TA = ri[WS(ris, 62)];
+		    TB = Tz + TA;
+		    T90 = Tz - TA;
+		    {
+			 E T3c, T3d, TC, TD;
+			 T3c = ii[WS(iis, 2)];
+			 T3d = ii[WS(iis, 62)];
+			 T3e = T3c - T3d;
+			 T9y = T3c + T3d;
+			 TC = ri[WS(ris, 34)];
+			 TD = ri[WS(ris, 30)];
+			 TE = TC + TD;
+			 T9x = TC - TD;
+		    }
+		    T3f = ii[WS(iis, 34)];
+		    T3g = ii[WS(iis, 30)];
+		    T3h = T3f - T3g;
+		    T91 = T3f + T3g;
+		    {
+			 E TJ, TK, T96, T2G, T2H, T97;
+			 TJ = ri[WS(ris, 14)];
+			 TK = ri[WS(ris, 50)];
+			 T96 = TJ - TK;
+			 T2G = ii[WS(iis, 50)];
+			 T2H = ii[WS(iis, 14)];
+			 T97 = T2H + T2G;
+			 TL = TJ + TK;
+			 T9v = T96 + T97;
+			 T2I = T2G - T2H;
+			 T98 = T96 - T97;
+		    }
+		    {
+			 E TG, TH, T93, T2J, T2K, T94;
+			 TG = ri[WS(ris, 18)];
+			 TH = ri[WS(ris, 46)];
+			 T93 = TG - TH;
+			 T2J = ii[WS(iis, 18)];
+			 T2K = ii[WS(iis, 46)];
+			 T94 = T2J + T2K;
+			 TI = TG + TH;
+			 T9u = T93 + T94;
+			 T2L = T2J - T2K;
+			 T95 = T93 - T94;
+		    }
+	       }
+	       {
+		    E TF, TM, TcS, TcT;
+		    TF = TB + TE;
+		    TM = TI + TL;
+		    TN = TF + TM;
+		    T6z = TF - TM;
+		    TcS = T90 + T91;
+		    TcT = KP707106781 * (T9u + T9v);
+		    TcU = TcS - TcT;
+		    Teu = TcS + TcT;
+	       }
+	       {
+		    E Td6, Td7, T2F, T2M;
+		    Td6 = KP707106781 * (T95 - T98);
+		    Td7 = T9y - T9x;
+		    Td8 = Td6 + Td7;
+		    Tey = Td7 - Td6;
+		    T2F = TB - TE;
+		    T2M = T2I - T2L;
+		    T2N = T2F + T2M;
+		    T5r = T2F - T2M;
+	       }
+	       {
+		    E T3b, T3i, T92, T99;
+		    T3b = TI - TL;
+		    T3i = T3e - T3h;
+		    T3j = T3b + T3i;
+		    T5v = T3i - T3b;
+		    T92 = T90 - T91;
+		    T99 = KP707106781 * (T95 + T98);
+		    T9a = T92 + T99;
+		    TbE = T92 - T99;
+	       }
+	       {
+		    E T9w, T9z, T6F, T6G;
+		    T9w = KP707106781 * (T9u - T9v);
+		    T9z = T9x + T9y;
+		    T9A = T9w + T9z;
+		    TbI = T9z - T9w;
+		    T6F = T3h + T3e;
+		    T6G = T2L + T2I;
+		    T6H = T6F - T6G;
+		    T7O = T6G + T6F;
+	       }
+	  }
+	  {
+	       E T1G, Taj, T3Q, Ta5, T46, Tak, T6R, Ta6, T1N, Tag, Tah, T3X, T3Z, Taa, Tad;
+	       E T6S, Tdn, Tdo;
+	       {
+		    E T1A, T1B, T1C, T1D, T1E, T1F;
+		    T1A = ri[WS(ris, 5)];
+		    T1B = ri[WS(ris, 59)];
+		    T1C = T1A + T1B;
+		    T1D = ri[WS(ris, 37)];
+		    T1E = ri[WS(ris, 27)];
+		    T1F = T1D + T1E;
+		    T1G = T1C + T1F;
+		    Taj = T1D - T1E;
+		    T3Q = T1C - T1F;
+		    Ta5 = T1A - T1B;
+	       }
+	       {
+		    E T40, T41, T42, T43, T44, T45;
+		    T40 = ii[WS(iis, 5)];
+		    T41 = ii[WS(iis, 59)];
+		    T42 = T40 - T41;
+		    T43 = ii[WS(iis, 37)];
+		    T44 = ii[WS(iis, 27)];
+		    T45 = T43 - T44;
+		    T46 = T42 - T45;
+		    Tak = T40 + T41;
+		    T6R = T45 + T42;
+		    Ta6 = T43 + T44;
+	       }
+	       {
+		    E T1J, Ta8, T3W, Ta9, T1M, Tab, T3T, Tac;
+		    {
+			 E T1H, T1I, T3U, T3V;
+			 T1H = ri[WS(ris, 21)];
+			 T1I = ri[WS(ris, 43)];
+			 T1J = T1H + T1I;
+			 Ta8 = T1H - T1I;
+			 T3U = ii[WS(iis, 21)];
+			 T3V = ii[WS(iis, 43)];
+			 T3W = T3U - T3V;
+			 Ta9 = T3U + T3V;
+		    }
+		    {
+			 E T1K, T1L, T3R, T3S;
+			 T1K = ri[WS(ris, 11)];
+			 T1L = ri[WS(ris, 53)];
+			 T1M = T1K + T1L;
+			 Tab = T1K - T1L;
+			 T3R = ii[WS(iis, 53)];
+			 T3S = ii[WS(iis, 11)];
+			 T3T = T3R - T3S;
+			 Tac = T3S + T3R;
+		    }
+		    T1N = T1J + T1M;
+		    Tag = Ta8 + Ta9;
+		    Tah = Tab + Tac;
+		    T3X = T3T - T3W;
+		    T3Z = T1J - T1M;
+		    Taa = Ta8 - Ta9;
+		    Tad = Tab - Tac;
+		    T6S = T3W + T3T;
+	       }
+	       T1O = T1G + T1N;
+	       T7V = T6S + T6R;
+	       {
+		    E T3Y, T47, Tdq, Tdr;
+		    T3Y = T3Q + T3X;
+		    T47 = T3Z + T46;
+		    T48 = FNMS(KP382683432, T47, KP923879532 * T3Y);
+		    T4u = FMA(KP382683432, T3Y, KP923879532 * T47);
+		    Tdq = KP707106781 * (Taa - Tad);
+		    Tdr = Tak - Taj;
+		    Tds = Tdq + Tdr;
+		    TeG = Tdr - Tdq;
+	       }
+	       {
+		    E T5C, T5D, Ta7, Tae;
+		    T5C = T3Q - T3X;
+		    T5D = T46 - T3Z;
+		    T5E = FNMS(KP923879532, T5D, KP382683432 * T5C);
+		    T5K = FMA(KP923879532, T5C, KP382683432 * T5D);
+		    Ta7 = Ta5 - Ta6;
+		    Tae = KP707106781 * (Taa + Tad);
+		    Taf = Ta7 + Tae;
+		    TbP = Ta7 - Tae;
+	       }
+	       Tdn = Ta5 + Ta6;
+	       Tdo = KP707106781 * (Tag + Tah);
+	       Tdp = Tdn - Tdo;
+	       TeF = Tdn + Tdo;
+	       {
+		    E T6Q, T6T, Tai, Tal;
+		    T6Q = T1G - T1N;
+		    T6T = T6R - T6S;
+		    T6U = T6Q - T6T;
+		    T72 = T6Q + T6T;
+		    Tai = KP707106781 * (Tag - Tah);
+		    Tal = Taj + Tak;
+		    Tam = Tai + Tal;
+		    TbQ = Tal - Tai;
+	       }
+	  }
+	  {
+	       E T1V, TaC, T49, Tao, T4p, TaD, T6W, Tap, T22, Taz, TaA, T4g, T4i, Tat, Taw;
+	       E T6X, Tdu, Tdv;
+	       {
+		    E T1P, T1Q, T1R, T1S, T1T, T1U;
+		    T1P = ri[WS(ris, 3)];
+		    T1Q = ri[WS(ris, 61)];
+		    T1R = T1P + T1Q;
+		    T1S = ri[WS(ris, 29)];
+		    T1T = ri[WS(ris, 35)];
+		    T1U = T1S + T1T;
+		    T1V = T1R + T1U;
+		    TaC = T1S - T1T;
+		    T49 = T1R - T1U;
+		    Tao = T1P - T1Q;
+	       }
+	       {
+		    E T4j, T4k, T4l, T4m, T4n, T4o;
+		    T4j = ii[WS(iis, 61)];
+		    T4k = ii[WS(iis, 3)];
+		    T4l = T4j - T4k;
+		    T4m = ii[WS(iis, 29)];
+		    T4n = ii[WS(iis, 35)];
+		    T4o = T4m - T4n;
+		    T4p = T4l - T4o;
+		    TaD = T4k + T4j;
+		    T6W = T4o + T4l;
+		    Tap = T4m + T4n;
+	       }
+	       {
+		    E T1Y, Tar, T4f, Tas, T21, Tau, T4c, Tav;
+		    {
+			 E T1W, T1X, T4d, T4e;
+			 T1W = ri[WS(ris, 13)];
+			 T1X = ri[WS(ris, 51)];
+			 T1Y = T1W + T1X;
+			 Tar = T1W - T1X;
+			 T4d = ii[WS(iis, 13)];
+			 T4e = ii[WS(iis, 51)];
+			 T4f = T4d - T4e;
+			 Tas = T4d + T4e;
+		    }
+		    {
+			 E T1Z, T20, T4a, T4b;
+			 T1Z = ri[WS(ris, 19)];
+			 T20 = ri[WS(ris, 45)];
+			 T21 = T1Z + T20;
+			 Tau = T1Z - T20;
+			 T4a = ii[WS(iis, 45)];
+			 T4b = ii[WS(iis, 19)];
+			 T4c = T4a - T4b;
+			 Tav = T4b + T4a;
+		    }
+		    T22 = T1Y + T21;
+		    Taz = Tar + Tas;
+		    TaA = Tau + Tav;
+		    T4g = T4c - T4f;
+		    T4i = T1Y - T21;
+		    Tat = Tar - Tas;
+		    Taw = Tau - Tav;
+		    T6X = T4f + T4c;
+	       }
+	       T23 = T1V + T22;
+	       T7U = T6X + T6W;
+	       {
+		    E T4h, T4q, Tdx, Tdy;
+		    T4h = T49 + T4g;
+		    T4q = T4i + T4p;
+		    T4r = FMA(KP923879532, T4h, KP382683432 * T4q);
+		    T4v = FNMS(KP382683432, T4h, KP923879532 * T4q);
+		    Tdx = KP707106781 * (Tat - Taw);
+		    Tdy = TaC + TaD;
+		    Tdz = Tdx - Tdy;
+		    TeJ = Tdx + Tdy;
+	       }
+	       {
+		    E T5F, T5G, Taq, Tax;
+		    T5F = T49 - T4g;
+		    T5G = T4p - T4i;
+		    T5H = FMA(KP382683432, T5F, KP923879532 * T5G);
+		    T5L = FNMS(KP923879532, T5F, KP382683432 * T5G);
+		    Taq = Tao - Tap;
+		    Tax = KP707106781 * (Tat + Taw);
+		    Tay = Taq + Tax;
+		    TbS = Taq - Tax;
+	       }
+	       Tdu = Tao + Tap;
+	       Tdv = KP707106781 * (Taz + TaA);
+	       Tdw = Tdu - Tdv;
+	       TeI = Tdu + Tdv;
+	       {
+		    E T6V, T6Y, TaB, TaE;
+		    T6V = T1V - T22;
+		    T6Y = T6W - T6X;
+		    T6Z = T6V + T6Y;
+		    T73 = T6Y - T6V;
+		    TaB = KP707106781 * (Taz - TaA);
+		    TaE = TaC - TaD;
+		    TaF = TaB + TaE;
+		    TbT = TaE - TaB;
+	       }
+	  }
+	  {
+	       E T1m, T3z, T1p, T3C, T3w, T3D, Tdg, Tdf, T9U, T9R, T1t, T3I, T1w, T3L, T3F;
+	       E T3M, Tdj, Tdi, Ta1, T9Y;
+	       {
+		    E T9P, T9T, T9S, T9Q;
+		    {
+			 E T1k, T1l, T3x, T3y;
+			 T1k = ri[WS(ris, 9)];
+			 T1l = ri[WS(ris, 55)];
+			 T1m = T1k + T1l;
+			 T9P = T1k - T1l;
+			 T3x = ii[WS(iis, 9)];
+			 T3y = ii[WS(iis, 55)];
+			 T3z = T3x - T3y;
+			 T9T = T3x + T3y;
+		    }
+		    {
+			 E T1n, T1o, T3A, T3B;
+			 T1n = ri[WS(ris, 41)];
+			 T1o = ri[WS(ris, 23)];
+			 T1p = T1n + T1o;
+			 T9S = T1n - T1o;
+			 T3A = ii[WS(iis, 41)];
+			 T3B = ii[WS(iis, 23)];
+			 T3C = T3A - T3B;
+			 T9Q = T3A + T3B;
+		    }
+		    T3w = T1m - T1p;
+		    T3D = T3z - T3C;
+		    Tdg = T9T - T9S;
+		    Tdf = T9P + T9Q;
+		    T9U = T9S + T9T;
+		    T9R = T9P - T9Q;
+	       }
+	       {
+		    E T9W, Ta0, T9Z, T9X;
+		    {
+			 E T1r, T1s, T3G, T3H;
+			 T1r = ri[WS(ris, 7)];
+			 T1s = ri[WS(ris, 57)];
+			 T1t = T1r + T1s;
+			 T9W = T1r - T1s;
+			 T3G = ii[WS(iis, 57)];
+			 T3H = ii[WS(iis, 7)];
+			 T3I = T3G - T3H;
+			 Ta0 = T3H + T3G;
+		    }
+		    {
+			 E T1u, T1v, T3J, T3K;
+			 T1u = ri[WS(ris, 25)];
+			 T1v = ri[WS(ris, 39)];
+			 T1w = T1u + T1v;
+			 T9Z = T1u - T1v;
+			 T3J = ii[WS(iis, 25)];
+			 T3K = ii[WS(iis, 39)];
+			 T3L = T3J - T3K;
+			 T9X = T3J + T3K;
+		    }
+		    T3F = T1t - T1w;
+		    T3M = T3I - T3L;
+		    Tdj = T9Z + Ta0;
+		    Tdi = T9W + T9X;
+		    Ta1 = T9Z - Ta0;
+		    T9Y = T9W - T9X;
+	       }
+	       {
+		    E T1q, T1x, Tdh, Tdk;
+		    T1q = T1m + T1p;
+		    T1x = T1t + T1w;
+		    T1y = T1q + T1x;
+		    T75 = T1q - T1x;
+		    Tdh = FNMS(KP923879532, Tdg, KP382683432 * Tdf);
+		    Tdk = FNMS(KP923879532, Tdj, KP382683432 * Tdi);
+		    Tdl = Tdh + Tdk;
+		    TeQ = Tdh - Tdk;
+	       }
+	       {
+		    E TdG, TdH, T3E, T3N;
+		    TdG = FMA(KP923879532, Tdf, KP382683432 * Tdg);
+		    TdH = FMA(KP923879532, Tdi, KP382683432 * Tdj);
+		    TdI = TdG - TdH;
+		    TeD = TdG + TdH;
+		    T3E = T3w - T3D;
+		    T3N = T3F + T3M;
+		    T3O = KP707106781 * (T3E + T3N);
+		    T5N = KP707106781 * (T3E - T3N);
+	       }
+	       {
+		    E T4x, T4y, T9V, Ta2;
+		    T4x = T3w + T3D;
+		    T4y = T3M - T3F;
+		    T4z = KP707106781 * (T4x + T4y);
+		    T5A = KP707106781 * (T4y - T4x);
+		    T9V = FNMS(KP382683432, T9U, KP923879532 * T9R);
+		    Ta2 = FMA(KP923879532, T9Y, KP382683432 * Ta1);
+		    Ta3 = T9V + Ta2;
+		    Tc0 = T9V - Ta2;
+	       }
+	       {
+		    E TaM, TaN, T6M, T6N;
+		    TaM = FMA(KP382683432, T9R, KP923879532 * T9U);
+		    TaN = FNMS(KP382683432, T9Y, KP923879532 * Ta1);
+		    TaO = TaM + TaN;
+		    TbN = TaN - TaM;
+		    T6M = T3L + T3I;
+		    T6N = T3C + T3z;
+		    T6O = T6M - T6N;
+		    T80 = T6N + T6M;
+	       }
+	  }
+	  {
+	       E TQ, T2R, TT, T2U, T2O, T2V, TcW, TcV, T9g, T9d, TX, T30, T10, T33, T2X;
+	       E T34, TcZ, TcY, T9n, T9k;
+	       {
+		    E T9b, T9f, T9e, T9c;
+		    {
+			 E TO, TP, T2P, T2Q;
+			 TO = ri[WS(ris, 10)];
+			 TP = ri[WS(ris, 54)];
+			 TQ = TO + TP;
+			 T9b = TO - TP;
+			 T2P = ii[WS(iis, 10)];
+			 T2Q = ii[WS(iis, 54)];
+			 T2R = T2P - T2Q;
+			 T9f = T2P + T2Q;
+		    }
+		    {
+			 E TR, TS, T2S, T2T;
+			 TR = ri[WS(ris, 42)];
+			 TS = ri[WS(ris, 22)];
+			 TT = TR + TS;
+			 T9e = TR - TS;
+			 T2S = ii[WS(iis, 42)];
+			 T2T = ii[WS(iis, 22)];
+			 T2U = T2S - T2T;
+			 T9c = T2S + T2T;
+		    }
+		    T2O = TQ - TT;
+		    T2V = T2R - T2U;
+		    TcW = T9f - T9e;
+		    TcV = T9b + T9c;
+		    T9g = T9e + T9f;
+		    T9d = T9b - T9c;
+	       }
+	       {
+		    E T9i, T9m, T9l, T9j;
+		    {
+			 E TV, TW, T2Y, T2Z;
+			 TV = ri[WS(ris, 6)];
+			 TW = ri[WS(ris, 58)];
+			 TX = TV + TW;
+			 T9i = TV - TW;
+			 T2Y = ii[WS(iis, 58)];
+			 T2Z = ii[WS(iis, 6)];
+			 T30 = T2Y - T2Z;
+			 T9m = T2Z + T2Y;
+		    }
+		    {
+			 E TY, TZ, T31, T32;
+			 TY = ri[WS(ris, 26)];
+			 TZ = ri[WS(ris, 38)];
+			 T10 = TY + TZ;
+			 T9l = TY - TZ;
+			 T31 = ii[WS(iis, 26)];
+			 T32 = ii[WS(iis, 38)];
+			 T33 = T31 - T32;
+			 T9j = T31 + T32;
+		    }
+		    T2X = TX - T10;
+		    T34 = T30 - T33;
+		    TcZ = T9l + T9m;
+		    TcY = T9i + T9j;
+		    T9n = T9l - T9m;
+		    T9k = T9i - T9j;
+	       }
+	       {
+		    E TU, T11, TcX, Td0;
+		    TU = TQ + TT;
+		    T11 = TX + T10;
+		    T12 = TU + T11;
+		    T6E = TU - T11;
+		    TcX = FNMS(KP923879532, TcW, KP382683432 * TcV);
+		    Td0 = FNMS(KP923879532, TcZ, KP382683432 * TcY);
+		    Td1 = TcX + Td0;
+		    Tex = TcX - Td0;
+	       }
+	       {
+		    E Td3, Td4, T2W, T35;
+		    Td3 = FMA(KP923879532, TcV, KP382683432 * TcW);
+		    Td4 = FMA(KP923879532, TcY, KP382683432 * TcZ);
+		    Td5 = Td3 - Td4;
+		    Tev = Td3 + Td4;
+		    T2W = T2O - T2V;
+		    T35 = T2X + T34;
+		    T36 = KP707106781 * (T2W + T35);
+		    T5u = KP707106781 * (T2W - T35);
+	       }
+	       {
+		    E T38, T39, T9h, T9o;
+		    T38 = T2O + T2V;
+		    T39 = T34 - T2X;
+		    T3a = KP707106781 * (T38 + T39);
+		    T5s = KP707106781 * (T39 - T38);
+		    T9h = FNMS(KP382683432, T9g, KP923879532 * T9d);
+		    T9o = FMA(KP923879532, T9k, KP382683432 * T9n);
+		    T9p = T9h + T9o;
+		    TbH = T9h - T9o;
+	       }
+	       {
+		    E T9r, T9s, T6A, T6B;
+		    T9r = FMA(KP382683432, T9d, KP923879532 * T9g);
+		    T9s = FNMS(KP382683432, T9k, KP923879532 * T9n);
+		    T9t = T9r + T9s;
+		    TbF = T9s - T9r;
+		    T6A = T33 + T30;
+		    T6B = T2U + T2R;
+		    T6C = T6A - T6B;
+		    T7P = T6B + T6A;
+	       }
+	  }
+	  {
+	       E T13, T8f, Ty, T8e, T25, T8h, T8k, T8p, Ti, T14, T8o;
+	       T13 = KP2_000000000 * (TN + T12);
+	       T8f = KP2_000000000 * (T7P + T7O);
+	       Ti = Ta + Th;
+	       Ty = Ti + Tx;
+	       T8e = Ti - Tx;
+	       {
+		    E T1z, T24, T8i, T8j;
+		    T1z = T1j + T1y;
+		    T24 = T1O + T23;
+		    T25 = KP2_000000000 * (T1z + T24);
+		    T8h = T1z - T24;
+		    T8i = T80 + T7Z;
+		    T8j = T7V + T7U;
+		    T8k = T8i - T8j;
+		    T8p = KP2_000000000 * (T8j + T8i);
+	       }
+	       T14 = Ty + T13;
+	       O[WS(os, 64)] = T14 - T25;
+	       O[0] = T14 + T25;
+	       T8o = Ty - T13;
+	       O[WS(os, 32)] = T8o - T8p;
+	       O[WS(os, 96)] = T8o + T8p;
+	       {
+		    E T8g, T8l, T8m, T8n;
+		    T8g = T8e - T8f;
+		    T8l = KP1_414213562 * (T8h - T8k);
+		    O[WS(os, 80)] = T8g - T8l;
+		    O[WS(os, 16)] = T8g + T8l;
+		    T8m = T8e + T8f;
+		    T8n = KP1_414213562 * (T8h + T8k);
+		    O[WS(os, 48)] = T8m - T8n;
+		    O[WS(os, 112)] = T8m + T8n;
+	       }
+	  }
+	  {
+	       E T7M, T86, T82, T8a, T7R, T87, T7X, T89, T7K, T7Y, T81;
+	       T7K = Ta - Th;
+	       T7M = T7K - T7L;
+	       T86 = T7K + T7L;
+	       T7Y = T1O - T23;
+	       T81 = T7Z - T80;
+	       T82 = T7Y + T81;
+	       T8a = T81 - T7Y;
+	       {
+		    E T7N, T7Q, T7T, T7W;
+		    T7N = TN - T12;
+		    T7Q = T7O - T7P;
+		    T7R = KP1_414213562 * (T7N - T7Q);
+		    T87 = KP1_414213562 * (T7N + T7Q);
+		    T7T = T1j - T1y;
+		    T7W = T7U - T7V;
+		    T7X = T7T + T7W;
+		    T89 = T7T - T7W;
+	       }
+	       {
+		    E T7S, T83, T8c, T8d;
+		    T7S = T7M + T7R;
+		    T83 = FNMS(KP765366864, T82, KP1_847759065 * T7X);
+		    O[WS(os, 72)] = T7S - T83;
+		    O[WS(os, 8)] = T7S + T83;
+		    T8c = T86 + T87;
+		    T8d = FMA(KP1_847759065, T89, KP765366864 * T8a);
+		    O[WS(os, 56)] = T8c - T8d;
+		    O[WS(os, 120)] = T8c + T8d;
+	       }
+	       {
+		    E T84, T85, T88, T8b;
+		    T84 = T7M - T7R;
+		    T85 = FMA(KP765366864, T7X, KP1_847759065 * T82);
+		    O[WS(os, 40)] = T84 - T85;
+		    O[WS(os, 104)] = T84 + T85;
+		    T88 = T86 - T87;
+		    T8b = FNMS(KP1_847759065, T8a, KP765366864 * T89);
+		    O[WS(os, 88)] = T88 - T8b;
+		    O[WS(os, 24)] = T88 + T8b;
+	       }
+	  }
+	  {
+	       E T2E, T4O, T4K, T4S, T3l, T4P, T4t, T4R;
+	       {
+		    E T2k, T2D, T4w, T4J;
+		    T2k = T2a + T2j;
+		    T2D = FNMS(KP765366864, T2C, KP1_847759065 * T2t);
+		    T2E = T2k + T2D;
+		    T4O = T2k - T2D;
+		    T4w = T4u + T4v;
+		    T4J = T4z + T4I;
+		    T4K = T4w + T4J;
+		    T4S = T4J - T4w;
+	       }
+	       {
+		    E T37, T3k, T3P, T4s;
+		    T37 = T2N + T36;
+		    T3k = T3a + T3j;
+		    T3l = FNMS(KP390180644, T3k, KP1_961570560 * T37);
+		    T4P = FMA(KP390180644, T37, KP1_961570560 * T3k);
+		    T3P = T3v + T3O;
+		    T4s = T48 + T4r;
+		    T4t = T3P + T4s;
+		    T4R = T3P - T4s;
+	       }
+	       {
+		    E T3m, T4L, T4U, T4V;
+		    T3m = T2E + T3l;
+		    T4L = FNMS(KP196034280, T4K, KP1_990369453 * T4t);
+		    O[WS(os, 66)] = T3m - T4L;
+		    O[WS(os, 2)] = T3m + T4L;
+		    T4U = T4O + T4P;
+		    T4V = FMA(KP1_546020906, T4R, KP1_268786568 * T4S);
+		    O[WS(os, 50)] = T4U - T4V;
+		    O[WS(os, 114)] = T4U + T4V;
+	       }
+	       {
+		    E T4M, T4N, T4Q, T4T;
+		    T4M = T2E - T3l;
+		    T4N = FMA(KP196034280, T4t, KP1_990369453 * T4K);
+		    O[WS(os, 34)] = T4M - T4N;
+		    O[WS(os, 98)] = T4M + T4N;
+		    T4Q = T4O - T4P;
+		    T4T = FNMS(KP1_546020906, T4S, KP1_268786568 * T4R);
+		    O[WS(os, 82)] = T4Q - T4T;
+		    O[WS(os, 18)] = T4Q + T4T;
+	       }
+	  }
+	  {
+	       E T6y, T7e, T7a, T7i, T6J, T7f, T71, T7h;
+	       {
+		    E T6s, T6x, T74, T79;
+		    T6s = T6q - T6r;
+		    T6x = KP1_414213562 * (T6t - T6w);
+		    T6y = T6s + T6x;
+		    T7e = T6s - T6x;
+		    T74 = KP707106781 * (T72 + T73);
+		    T79 = T75 + T78;
+		    T7a = T74 + T79;
+		    T7i = T79 - T74;
+	       }
+	       {
+		    E T6D, T6I, T6P, T70;
+		    T6D = T6z + T6C;
+		    T6I = T6E + T6H;
+		    T6J = FNMS(KP765366864, T6I, KP1_847759065 * T6D);
+		    T7f = FMA(KP765366864, T6D, KP1_847759065 * T6I);
+		    T6P = T6L + T6O;
+		    T70 = KP707106781 * (T6U + T6Z);
+		    T71 = T6P + T70;
+		    T7h = T6P - T70;
+	       }
+	       {
+		    E T6K, T7b, T7k, T7l;
+		    T6K = T6y + T6J;
+		    T7b = FNMS(KP390180644, T7a, KP1_961570560 * T71);
+		    O[WS(os, 68)] = T6K - T7b;
+		    O[WS(os, 4)] = T6K + T7b;
+		    T7k = T7e + T7f;
+		    T7l = FMA(KP1_662939224, T7h, KP1_111140466 * T7i);
+		    O[WS(os, 52)] = T7k - T7l;
+		    O[WS(os, 116)] = T7k + T7l;
+	       }
+	       {
+		    E T7c, T7d, T7g, T7j;
+		    T7c = T6y - T6J;
+		    T7d = FMA(KP390180644, T71, KP1_961570560 * T7a);
+		    O[WS(os, 36)] = T7c - T7d;
+		    O[WS(os, 100)] = T7c + T7d;
+		    T7g = T7e - T7f;
+		    T7j = FNMS(KP1_662939224, T7i, KP1_111140466 * T7h);
+		    O[WS(os, 84)] = T7g - T7j;
+		    O[WS(os, 20)] = T7g + T7j;
+	       }
+	  }
+	  {
+	       E T4Y, T5c, T58, T5g, T51, T5d, T55, T5f;
+	       {
+		    E T4W, T4X, T56, T57;
+		    T4W = T2a - T2j;
+		    T4X = FMA(KP765366864, T2t, KP1_847759065 * T2C);
+		    T4Y = T4W - T4X;
+		    T5c = T4W + T4X;
+		    T56 = T48 - T4r;
+		    T57 = T4I - T4z;
+		    T58 = T56 + T57;
+		    T5g = T57 - T56;
+	       }
+	       {
+		    E T4Z, T50, T53, T54;
+		    T4Z = T2N - T36;
+		    T50 = T3j - T3a;
+		    T51 = FNMS(KP1_662939224, T50, KP1_111140466 * T4Z);
+		    T5d = FMA(KP1_662939224, T4Z, KP1_111140466 * T50);
+		    T53 = T3v - T3O;
+		    T54 = T4v - T4u;
+		    T55 = T53 + T54;
+		    T5f = T53 - T54;
+	       }
+	       {
+		    E T52, T59, T5i, T5j;
+		    T52 = T4Y + T51;
+		    T59 = FNMS(KP942793473, T58, KP1_763842528 * T55);
+		    O[WS(os, 74)] = T52 - T59;
+		    O[WS(os, 10)] = T52 + T59;
+		    T5i = T5c + T5d;
+		    T5j = FMA(KP1_913880671, T5f, KP580569354 * T5g);
+		    O[WS(os, 58)] = T5i - T5j;
+		    O[WS(os, 122)] = T5i + T5j;
+	       }
+	       {
+		    E T5a, T5b, T5e, T5h;
+		    T5a = T4Y - T51;
+		    T5b = FMA(KP942793473, T55, KP1_763842528 * T58);
+		    O[WS(os, 42)] = T5a - T5b;
+		    O[WS(os, 106)] = T5a + T5b;
+		    T5e = T5c - T5d;
+		    T5h = FNMS(KP1_913880671, T5g, KP580569354 * T5f);
+		    O[WS(os, 90)] = T5e - T5h;
+		    O[WS(os, 26)] = T5e + T5h;
+	       }
+	  }
+	  {
+	       E T7o, T7C, T7y, T7G, T7r, T7D, T7v, T7F;
+	       {
+		    E T7m, T7n, T7w, T7x;
+		    T7m = T6q + T6r;
+		    T7n = KP1_414213562 * (T6t + T6w);
+		    T7o = T7m - T7n;
+		    T7C = T7m + T7n;
+		    T7w = KP707106781 * (T6U - T6Z);
+		    T7x = T78 - T75;
+		    T7y = T7w + T7x;
+		    T7G = T7x - T7w;
+	       }
+	       {
+		    E T7p, T7q, T7t, T7u;
+		    T7p = T6z - T6C;
+		    T7q = T6H - T6E;
+		    T7r = FNMS(KP1_847759065, T7q, KP765366864 * T7p);
+		    T7D = FMA(KP1_847759065, T7p, KP765366864 * T7q);
+		    T7t = T6L - T6O;
+		    T7u = KP707106781 * (T73 - T72);
+		    T7v = T7t + T7u;
+		    T7F = T7t - T7u;
+	       }
+	       {
+		    E T7s, T7z, T7I, T7J;
+		    T7s = T7o + T7r;
+		    T7z = FNMS(KP1_111140466, T7y, KP1_662939224 * T7v);
+		    O[WS(os, 76)] = T7s - T7z;
+		    O[WS(os, 12)] = T7s + T7z;
+		    T7I = T7C + T7D;
+		    T7J = FMA(KP1_961570560, T7F, KP390180644 * T7G);
+		    O[WS(os, 60)] = T7I - T7J;
+		    O[WS(os, 124)] = T7I + T7J;
+	       }
+	       {
+		    E T7A, T7B, T7E, T7H;
+		    T7A = T7o - T7r;
+		    T7B = FMA(KP1_111140466, T7v, KP1_662939224 * T7y);
+		    O[WS(os, 44)] = T7A - T7B;
+		    O[WS(os, 108)] = T7A + T7B;
+		    T7E = T7C - T7D;
+		    T7H = FNMS(KP1_961570560, T7G, KP390180644 * T7F);
+		    O[WS(os, 92)] = T7E - T7H;
+		    O[WS(os, 28)] = T7E + T7H;
+	       }
+	  }
+	  {
+	       E T5q, T5U, T5Q, T5Y, T5x, T5V, T5J, T5X;
+	       {
+		    E T5m, T5p, T5M, T5P;
+		    T5m = T5k - T5l;
+		    T5p = FNMS(KP1_847759065, T5o, KP765366864 * T5n);
+		    T5q = T5m + T5p;
+		    T5U = T5m - T5p;
+		    T5M = T5K + T5L;
+		    T5P = T5N + T5O;
+		    T5Q = T5M + T5P;
+		    T5Y = T5P - T5M;
+	       }
+	       {
+		    E T5t, T5w, T5B, T5I;
+		    T5t = T5r + T5s;
+		    T5w = T5u + T5v;
+		    T5x = FNMS(KP1_111140466, T5w, KP1_662939224 * T5t);
+		    T5V = FMA(KP1_111140466, T5t, KP1_662939224 * T5w);
+		    T5B = T5z + T5A;
+		    T5I = T5E + T5H;
+		    T5J = T5B + T5I;
+		    T5X = T5B - T5I;
+	       }
+	       {
+		    E T5y, T5R, T60, T61;
+		    T5y = T5q + T5x;
+		    T5R = FNMS(KP580569354, T5Q, KP1_913880671 * T5J);
+		    O[WS(os, 70)] = T5y - T5R;
+		    O[WS(os, 6)] = T5y + T5R;
+		    T60 = T5U + T5V;
+		    T61 = FMA(KP1_763842528, T5X, KP942793473 * T5Y);
+		    O[WS(os, 54)] = T60 - T61;
+		    O[WS(os, 118)] = T60 + T61;
+	       }
+	       {
+		    E T5S, T5T, T5W, T5Z;
+		    T5S = T5q - T5x;
+		    T5T = FMA(KP580569354, T5J, KP1_913880671 * T5Q);
+		    O[WS(os, 38)] = T5S - T5T;
+		    O[WS(os, 102)] = T5S + T5T;
+		    T5W = T5U - T5V;
+		    T5Z = FNMS(KP1_763842528, T5Y, KP942793473 * T5X);
+		    O[WS(os, 86)] = T5W - T5Z;
+		    O[WS(os, 22)] = T5W + T5Z;
+	       }
+	  }
+	  {
+	       E T64, T6i, T6e, T6m, T67, T6j, T6b, T6l;
+	       {
+		    E T62, T63, T6c, T6d;
+		    T62 = T5k + T5l;
+		    T63 = FMA(KP1_847759065, T5n, KP765366864 * T5o);
+		    T64 = T62 - T63;
+		    T6i = T62 + T63;
+		    T6c = T5E - T5H;
+		    T6d = T5O - T5N;
+		    T6e = T6c + T6d;
+		    T6m = T6d - T6c;
+	       }
+	       {
+		    E T65, T66, T69, T6a;
+		    T65 = T5r - T5s;
+		    T66 = T5v - T5u;
+		    T67 = FNMS(KP1_961570560, T66, KP390180644 * T65);
+		    T6j = FMA(KP1_961570560, T65, KP390180644 * T66);
+		    T69 = T5z - T5A;
+		    T6a = T5L - T5K;
+		    T6b = T69 + T6a;
+		    T6l = T69 - T6a;
+	       }
+	       {
+		    E T68, T6f, T6o, T6p;
+		    T68 = T64 + T67;
+		    T6f = FNMS(KP1_268786568, T6e, KP1_546020906 * T6b);
+		    O[WS(os, 78)] = T68 - T6f;
+		    O[WS(os, 14)] = T68 + T6f;
+		    T6o = T6i + T6j;
+		    T6p = FMA(KP1_990369453, T6l, KP196034280 * T6m);
+		    O[WS(os, 62)] = T6o - T6p;
+		    O[WS(os, 126)] = T6o + T6p;
+	       }
+	       {
+		    E T6g, T6h, T6k, T6n;
+		    T6g = T64 - T67;
+		    T6h = FMA(KP1_268786568, T6b, KP1_546020906 * T6e);
+		    O[WS(os, 46)] = T6g - T6h;
+		    O[WS(os, 110)] = T6g + T6h;
+		    T6k = T6i - T6j;
+		    T6n = FNMS(KP1_990369453, T6m, KP196034280 * T6l);
+		    O[WS(os, 94)] = T6k - T6n;
+		    O[WS(os, 30)] = T6k + T6n;
+	       }
+	  }
+	  {
+	       E T8Z, Tb1, T9C, Tb2, Tbe, Tbq, Tbb, Tbp, TaX, Tbs, Tb5, Tbi, TaI, Tbt, Tb4;
+	       E Tbl;
+	       {
+		    E T8F, T8Y, Tb9, Tba;
+		    T8F = T8x + T8E;
+		    T8Y = FNMS(KP390180644, T8X, KP1_961570560 * T8Q);
+		    T8Z = T8F + T8Y;
+		    Tb1 = T8F - T8Y;
+		    {
+			 E T9q, T9B, Tbc, Tbd;
+			 T9q = T9a + T9p;
+			 T9B = T9t + T9A;
+			 T9C = FNMS(KP196034280, T9B, KP1_990369453 * T9q);
+			 Tb2 = FMA(KP196034280, T9q, KP1_990369453 * T9B);
+			 Tbc = T9a - T9p;
+			 Tbd = T9A - T9t;
+			 Tbe = FNMS(KP1_546020906, Tbd, KP1_268786568 * Tbc);
+			 Tbq = FMA(KP1_546020906, Tbc, KP1_268786568 * Tbd);
+		    }
+		    Tb9 = T8x - T8E;
+		    Tba = FMA(KP390180644, T8Q, KP1_961570560 * T8X);
+		    Tbb = Tb9 - Tba;
+		    Tbp = Tb9 + Tba;
+		    {
+			 E TaW, Tbg, TaL, Tbh, TaJ, TaK;
+			 TaW = TaO + TaV;
+			 Tbg = T9O - Ta3;
+			 TaJ = FMA(KP195090322, Taf, KP980785280 * Tam);
+			 TaK = FNMS(KP195090322, Tay, KP980785280 * TaF);
+			 TaL = TaJ + TaK;
+			 Tbh = TaK - TaJ;
+			 TaX = TaL + TaW;
+			 Tbs = Tbg - Tbh;
+			 Tb5 = TaW - TaL;
+			 Tbi = Tbg + Tbh;
+		    }
+		    {
+			 E Ta4, Tbk, TaH, Tbj, Tan, TaG;
+			 Ta4 = T9O + Ta3;
+			 Tbk = TaV - TaO;
+			 Tan = FNMS(KP195090322, Tam, KP980785280 * Taf);
+			 TaG = FMA(KP980785280, Tay, KP195090322 * TaF);
+			 TaH = Tan + TaG;
+			 Tbj = Tan - TaG;
+			 TaI = Ta4 + TaH;
+			 Tbt = Tbk - Tbj;
+			 Tb4 = Ta4 - TaH;
+			 Tbl = Tbj + Tbk;
+		    }
+	       }
+	       {
+		    E T9D, TaY, Tbr, Tbu;
+		    T9D = T8Z + T9C;
+		    TaY = FNMS(KP098135348, TaX, KP1_997590912 * TaI);
+		    O[WS(os, 65)] = T9D - TaY;
+		    O[WS(os, 1)] = T9D + TaY;
+		    Tbr = Tbp - Tbq;
+		    Tbu = FNMS(KP1_883088130, Tbt, KP673779706 * Tbs);
+		    O[WS(os, 89)] = Tbr - Tbu;
+		    O[WS(os, 25)] = Tbr + Tbu;
+	       }
+	       {
+		    E Tbv, Tbw, TaZ, Tb0;
+		    Tbv = Tbp + Tbq;
+		    Tbw = FMA(KP1_883088130, Tbs, KP673779706 * Tbt);
+		    O[WS(os, 57)] = Tbv - Tbw;
+		    O[WS(os, 121)] = Tbv + Tbw;
+		    TaZ = T8Z - T9C;
+		    Tb0 = FMA(KP098135348, TaI, KP1_997590912 * TaX);
+		    O[WS(os, 33)] = TaZ - Tb0;
+		    O[WS(os, 97)] = TaZ + Tb0;
+	       }
+	       {
+		    E Tb3, Tb6, Tbf, Tbm;
+		    Tb3 = Tb1 - Tb2;
+		    Tb6 = FNMS(KP1_481902250, Tb5, KP1_343117909 * Tb4);
+		    O[WS(os, 81)] = Tb3 - Tb6;
+		    O[WS(os, 17)] = Tb3 + Tb6;
+		    Tbf = Tbb + Tbe;
+		    Tbm = FNMS(KP855110186, Tbl, KP1_807978586 * Tbi);
+		    O[WS(os, 73)] = Tbf - Tbm;
+		    O[WS(os, 9)] = Tbf + Tbm;
+	       }
+	       {
+		    E Tbn, Tbo, Tb7, Tb8;
+		    Tbn = Tbb - Tbe;
+		    Tbo = FMA(KP855110186, Tbi, KP1_807978586 * Tbl);
+		    O[WS(os, 41)] = Tbn - Tbo;
+		    O[WS(os, 105)] = Tbn + Tbo;
+		    Tb7 = Tb1 + Tb2;
+		    Tb8 = FMA(KP1_481902250, Tb4, KP1_343117909 * Tb5);
+		    O[WS(os, 49)] = Tb7 - Tb8;
+		    O[WS(os, 113)] = Tb7 + Tb8;
+	       }
+	  }
+	  {
+	       E TcR, TdR, Tda, TdS, Te4, Teg, Te1, Tef, TdN, Tei, TdV, Te8, TdC, Tej, TdU;
+	       E Teb;
+	       {
+		    E TcJ, TcQ, TdZ, Te0;
+		    TcJ = TcF + TcI;
+		    TcQ = FNMS(KP1_111140466, TcP, KP1_662939224 * TcM);
+		    TcR = TcJ + TcQ;
+		    TdR = TcJ - TcQ;
+		    {
+			 E Td2, Td9, Te2, Te3;
+			 Td2 = TcU + Td1;
+			 Td9 = Td5 + Td8;
+			 Tda = FNMS(KP580569354, Td9, KP1_913880671 * Td2);
+			 TdS = FMA(KP580569354, Td2, KP1_913880671 * Td9);
+			 Te2 = TcU - Td1;
+			 Te3 = Td8 - Td5;
+			 Te4 = FNMS(KP1_763842528, Te3, KP942793473 * Te2);
+			 Teg = FMA(KP1_763842528, Te2, KP942793473 * Te3);
+		    }
+		    TdZ = TcF - TcI;
+		    Te0 = FMA(KP1_111140466, TcM, KP1_662939224 * TcP);
+		    Te1 = TdZ - Te0;
+		    Tef = TdZ + Te0;
+		    {
+			 E TdM, Te6, TdF, Te7, TdD, TdE;
+			 TdM = TdI + TdL;
+			 Te6 = Tde - Tdl;
+			 TdD = FMA(KP555570233, Tdp, KP831469612 * Tds);
+			 TdE = FNMS(KP555570233, Tdw, KP831469612 * Tdz);
+			 TdF = TdD + TdE;
+			 Te7 = TdE - TdD;
+			 TdN = TdF + TdM;
+			 Tei = Te6 - Te7;
+			 TdV = TdM - TdF;
+			 Te8 = Te6 + Te7;
+		    }
+		    {
+			 E Tdm, Tea, TdB, Te9, Tdt, TdA;
+			 Tdm = Tde + Tdl;
+			 Tea = TdL - TdI;
+			 Tdt = FNMS(KP555570233, Tds, KP831469612 * Tdp);
+			 TdA = FMA(KP831469612, Tdw, KP555570233 * Tdz);
+			 TdB = Tdt + TdA;
+			 Te9 = Tdt - TdA;
+			 TdC = Tdm + TdB;
+			 Tej = Tea - Te9;
+			 TdU = Tdm - TdB;
+			 Teb = Te9 + Tea;
+		    }
+	       }
+	       {
+		    E Tdb, TdO, Teh, Tek;
+		    Tdb = TcR + Tda;
+		    TdO = FNMS(KP293460948, TdN, KP1_978353019 * TdC);
+		    O[WS(os, 67)] = Tdb - TdO;
+		    O[WS(os, 3)] = Tdb + TdO;
+		    Teh = Tef - Teg;
+		    Tek = FNMS(KP1_940062506, Tej, KP485960359 * Tei);
+		    O[WS(os, 91)] = Teh - Tek;
+		    O[WS(os, 27)] = Teh + Tek;
+	       }
+	       {
+		    E Tel, Tem, TdP, TdQ;
+		    Tel = Tef + Teg;
+		    Tem = FMA(KP1_940062506, Tei, KP485960359 * Tej);
+		    O[WS(os, 59)] = Tel - Tem;
+		    O[WS(os, 123)] = Tel + Tem;
+		    TdP = TcR - Tda;
+		    TdQ = FMA(KP293460948, TdC, KP1_978353019 * TdN);
+		    O[WS(os, 35)] = TdP - TdQ;
+		    O[WS(os, 99)] = TdP + TdQ;
+	       }
+	       {
+		    E TdT, TdW, Te5, Tec;
+		    TdT = TdR - TdS;
+		    TdW = FNMS(KP1_606415062, TdV, KP1_191398608 * TdU);
+		    O[WS(os, 83)] = TdT - TdW;
+		    O[WS(os, 19)] = TdT + TdW;
+		    Te5 = Te1 + Te4;
+		    Tec = FNMS(KP1_028205488, Teb, KP1_715457220 * Te8);
+		    O[WS(os, 75)] = Te5 - Tec;
+		    O[WS(os, 11)] = Te5 + Tec;
+	       }
+	       {
+		    E Ted, Tee, TdX, TdY;
+		    Ted = Te1 - Te4;
+		    Tee = FMA(KP1_028205488, Te8, KP1_715457220 * Teb);
+		    O[WS(os, 43)] = Ted - Tee;
+		    O[WS(os, 107)] = Ted + Tee;
+		    TdX = TdR + TdS;
+		    TdY = FMA(KP1_606415062, TdU, KP1_191398608 * TdV);
+		    O[WS(os, 51)] = TdX - TdY;
+		    O[WS(os, 115)] = TdX + TdY;
+	       }
+	  }
+	  {
+	       E TbD, Tc7, TbK, Tc8, Tck, Tcw, Tch, Tcv, Tc3, Tcy, Tcb, Tco, TbW, Tcz, Tca;
+	       E Tcr;
+	       {
+		    E Tbz, TbC, Tcf, Tcg;
+		    Tbz = Tbx - Tby;
+		    TbC = FNMS(KP1_662939224, TbB, KP1_111140466 * TbA);
+		    TbD = Tbz + TbC;
+		    Tc7 = Tbz - TbC;
+		    {
+			 E TbG, TbJ, Tci, Tcj;
+			 TbG = TbE + TbF;
+			 TbJ = TbH + TbI;
+			 TbK = FNMS(KP942793473, TbJ, KP1_763842528 * TbG);
+			 Tc8 = FMA(KP942793473, TbG, KP1_763842528 * TbJ);
+			 Tci = TbE - TbF;
+			 Tcj = TbI - TbH;
+			 Tck = FNMS(KP1_913880671, Tcj, KP580569354 * Tci);
+			 Tcw = FMA(KP1_913880671, Tci, KP580569354 * Tcj);
+		    }
+		    Tcf = Tbx + Tby;
+		    Tcg = FMA(KP1_662939224, TbA, KP1_111140466 * TbB);
+		    Tch = Tcf - Tcg;
+		    Tcv = Tcf + Tcg;
+		    {
+			 E Tc2, Tcm, TbZ, Tcn, TbX, TbY;
+			 Tc2 = Tc0 + Tc1;
+			 Tcm = TbM - TbN;
+			 TbX = FMA(KP831469612, TbP, KP555570233 * TbQ);
+			 TbY = FNMS(KP831469612, TbS, KP555570233 * TbT);
+			 TbZ = TbX + TbY;
+			 Tcn = TbY - TbX;
+			 Tc3 = TbZ + Tc2;
+			 Tcy = Tcm - Tcn;
+			 Tcb = Tc2 - TbZ;
+			 Tco = Tcm + Tcn;
+		    }
+		    {
+			 E TbO, Tcq, TbV, Tcp, TbR, TbU;
+			 TbO = TbM + TbN;
+			 Tcq = Tc1 - Tc0;
+			 TbR = FNMS(KP831469612, TbQ, KP555570233 * TbP);
+			 TbU = FMA(KP555570233, TbS, KP831469612 * TbT);
+			 TbV = TbR + TbU;
+			 Tcp = TbR - TbU;
+			 TbW = TbO + TbV;
+			 Tcz = Tcq - Tcp;
+			 Tca = TbO - TbV;
+			 Tcr = Tcp + Tcq;
+		    }
+	       }
+	       {
+		    E TbL, Tc4, Tcx, TcA;
+		    TbL = TbD + TbK;
+		    Tc4 = FNMS(KP485960359, Tc3, KP1_940062506 * TbW);
+		    O[WS(os, 69)] = TbL - Tc4;
+		    O[WS(os, 5)] = TbL + Tc4;
+		    Tcx = Tcv - Tcw;
+		    TcA = FNMS(KP1_978353019, Tcz, KP293460948 * Tcy);
+		    O[WS(os, 93)] = Tcx - TcA;
+		    O[WS(os, 29)] = Tcx + TcA;
+	       }
+	       {
+		    E TcB, TcC, Tc5, Tc6;
+		    TcB = Tcv + Tcw;
+		    TcC = FMA(KP1_978353019, Tcy, KP293460948 * Tcz);
+		    O[WS(os, 61)] = TcB - TcC;
+		    O[WS(os, 125)] = TcB + TcC;
+		    Tc5 = TbD - TbK;
+		    Tc6 = FMA(KP485960359, TbW, KP1_940062506 * Tc3);
+		    O[WS(os, 37)] = Tc5 - Tc6;
+		    O[WS(os, 101)] = Tc5 + Tc6;
+	       }
+	       {
+		    E Tc9, Tcc, Tcl, Tcs;
+		    Tc9 = Tc7 - Tc8;
+		    Tcc = FNMS(KP1_715457220, Tcb, KP1_028205488 * Tca);
+		    O[WS(os, 85)] = Tc9 - Tcc;
+		    O[WS(os, 21)] = Tc9 + Tcc;
+		    Tcl = Tch + Tck;
+		    Tcs = FNMS(KP1_191398608, Tcr, KP1_606415062 * Tco);
+		    O[WS(os, 77)] = Tcl - Tcs;
+		    O[WS(os, 13)] = Tcl + Tcs;
+	       }
+	       {
+		    E Tct, Tcu, Tcd, Tce;
+		    Tct = Tch - Tck;
+		    Tcu = FMA(KP1_191398608, Tco, KP1_606415062 * Tcr);
+		    O[WS(os, 45)] = Tct - Tcu;
+		    O[WS(os, 109)] = Tct + Tcu;
+		    Tcd = Tc7 + Tc8;
+		    Tce = FMA(KP1_715457220, Tca, KP1_028205488 * Tcb);
+		    O[WS(os, 53)] = Tcd - Tce;
+		    O[WS(os, 117)] = Tcd + Tce;
+	       }
+	  }
+	  {
+	       E Tet, TeX, TeA, TeY, Tfa, Tfm, Tf7, Tfl, TeT, Tfo, Tf1, Tfe, TeM, Tfp, Tf0;
+	       E Tfh;
+	       {
+		    E Tep, Tes, Tf5, Tf6;
+		    Tep = Ten - Teo;
+		    Tes = FNMS(KP1_961570560, Ter, KP390180644 * Teq);
+		    Tet = Tep + Tes;
+		    TeX = Tep - Tes;
+		    {
+			 E Tew, Tez, Tf8, Tf9;
+			 Tew = Teu - Tev;
+			 Tez = Tex + Tey;
+			 TeA = FNMS(KP1_268786568, Tez, KP1_546020906 * Tew);
+			 TeY = FMA(KP1_268786568, Tew, KP1_546020906 * Tez);
+			 Tf8 = Teu + Tev;
+			 Tf9 = Tey - Tex;
+			 Tfa = FNMS(KP1_990369453, Tf9, KP196034280 * Tf8);
+			 Tfm = FMA(KP1_990369453, Tf8, KP196034280 * Tf9);
+		    }
+		    Tf5 = Ten + Teo;
+		    Tf6 = FMA(KP1_961570560, Teq, KP390180644 * Ter);
+		    Tf7 = Tf5 - Tf6;
+		    Tfl = Tf5 + Tf6;
+		    {
+			 E TeS, Tfc, TeP, Tfd, TeN, TeO;
+			 TeS = TeQ + TeR;
+			 Tfc = TeC + TeD;
+			 TeN = FMA(KP980785280, TeF, KP195090322 * TeG);
+			 TeO = FMA(KP980785280, TeI, KP195090322 * TeJ);
+			 TeP = TeN - TeO;
+			 Tfd = TeN + TeO;
+			 TeT = TeP + TeS;
+			 Tfo = Tfc + Tfd;
+			 Tf1 = TeS - TeP;
+			 Tfe = Tfc - Tfd;
+		    }
+		    {
+			 E TeE, Tfg, TeL, Tff, TeH, TeK;
+			 TeE = TeC - TeD;
+			 Tfg = TeR - TeQ;
+			 TeH = FNMS(KP980785280, TeG, KP195090322 * TeF);
+			 TeK = FNMS(KP980785280, TeJ, KP195090322 * TeI);
+			 TeL = TeH + TeK;
+			 Tff = TeH - TeK;
+			 TeM = TeE + TeL;
+			 Tfp = Tfg - Tff;
+			 Tf0 = TeE - TeL;
+			 Tfh = Tff + Tfg;
+		    }
+	       }
+	       {
+		    E TeB, TeU, Tfn, Tfq;
+		    TeB = Tet + TeA;
+		    TeU = FNMS(KP673779706, TeT, KP1_883088130 * TeM);
+		    O[WS(os, 71)] = TeB - TeU;
+		    O[WS(os, 7)] = TeB + TeU;
+		    Tfn = Tfl - Tfm;
+		    Tfq = FNMS(KP1_997590912, Tfp, KP098135348 * Tfo);
+		    O[WS(os, 95)] = Tfn - Tfq;
+		    O[WS(os, 31)] = Tfn + Tfq;
+	       }
+	       {
+		    E Tfr, Tfs, TeV, TeW;
+		    Tfr = Tfl + Tfm;
+		    Tfs = FMA(KP1_997590912, Tfo, KP098135348 * Tfp);
+		    O[WS(os, 63)] = Tfr - Tfs;
+		    O[WS(os, 127)] = Tfr + Tfs;
+		    TeV = Tet - TeA;
+		    TeW = FMA(KP673779706, TeM, KP1_883088130 * TeT);
+		    O[WS(os, 39)] = TeV - TeW;
+		    O[WS(os, 103)] = TeV + TeW;
+	       }
+	       {
+		    E TeZ, Tf2, Tfb, Tfi;
+		    TeZ = TeX - TeY;
+		    Tf2 = FNMS(KP1_807978586, Tf1, KP855110186 * Tf0);
+		    O[WS(os, 87)] = TeZ - Tf2;
+		    O[WS(os, 23)] = TeZ + Tf2;
+		    Tfb = Tf7 + Tfa;
+		    Tfi = FNMS(KP1_343117909, Tfh, KP1_481902250 * Tfe);
+		    O[WS(os, 79)] = Tfb - Tfi;
+		    O[WS(os, 15)] = Tfb + Tfi;
+	       }
+	       {
+		    E Tfj, Tfk, Tf3, Tf4;
+		    Tfj = Tf7 - Tfa;
+		    Tfk = FMA(KP1_343117909, Tfe, KP1_481902250 * Tfh);
+		    O[WS(os, 47)] = Tfj - Tfk;
+		    O[WS(os, 111)] = Tfj + Tfk;
+		    Tf3 = TeX + TeY;
+		    Tf4 = FMA(KP1_807978586, Tf0, KP855110186 * Tf1);
+		    O[WS(os, 55)] = Tf3 - Tf4;
+		    O[WS(os, 119)] = Tf3 + Tf4;
+	       }
+	  }
+     }
+}
+
+static void mhc2r_128(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  mhc2r_128_0(ri, ii, O, ris, iis, os);
+	  ri += ivs;
+	  ii += ivs;
+	  O += ovs;
+     }
+}
+
+static const khc2r_desc desc = { 128, "mhc2r_128", {812, 198, 144, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_mhc2r_128) (planner *p) {
+     X(khc2r_register) (p, mhc2r_128, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/mhc2r_32.c b/src/fftw3/rdft/codelets/hc2r/mhc2r_32.c
new file mode 100644
index 0000000..154a6d4
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/mhc2r_32.c
@@ -0,0 +1,346 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:15 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r_noinline -compact -variables 4 -sign 1 -n 32 -name mhc2r_32 -include hc2r.h */
+
+/*
+ * This function contains 156 FP additions, 50 FP multiplications,
+ * (or, 140 additions, 34 multiplications, 16 fused multiply/add),
+ * 53 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: mhc2r_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: mhc2r_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: mhc2r_32.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void mhc2r_32_0(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os)
+{
+     DK(KP1_662939224, +1.662939224605090474157576755235811513477121624);
+     DK(KP1_111140466, +1.111140466039204449485661627897065748749874382);
+     DK(KP1_961570560, +1.961570560806460898252364472268478073947867462);
+     DK(KP390180644, +0.390180644032256535696569736954044481855383236);
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     {
+	  E T9, T2c, TB, T1y, T6, T2b, Ty, T1v, Th, T2e, T2f, TD, TK, T1C, T1F;
+	  E T1h, Tp, T2i, T2m, TN, T13, T1K, T1Y, T1k, Tw, TU, T1l, TW, T1V, T2j;
+	  E T1R, T2l;
+	  {
+	       E T7, T8, T1w, Tz, TA, T1x;
+	       T7 = ri[WS(ris, 4)];
+	       T8 = ri[WS(ris, 12)];
+	       T1w = T7 - T8;
+	       Tz = ii[WS(iis, 4)];
+	       TA = ii[WS(iis, 12)];
+	       T1x = Tz + TA;
+	       T9 = KP2_000000000 * (T7 + T8);
+	       T2c = KP1_414213562 * (T1w + T1x);
+	       TB = KP2_000000000 * (Tz - TA);
+	       T1y = KP1_414213562 * (T1w - T1x);
+	  }
+	  {
+	       E T5, T1u, T3, T1s;
+	       {
+		    E T4, T1t, T1, T2;
+		    T4 = ri[WS(ris, 8)];
+		    T5 = KP2_000000000 * T4;
+		    T1t = ii[WS(iis, 8)];
+		    T1u = KP2_000000000 * T1t;
+		    T1 = ri[0];
+		    T2 = ri[WS(ris, 16)];
+		    T3 = T1 + T2;
+		    T1s = T1 - T2;
+	       }
+	       T6 = T3 + T5;
+	       T2b = T1s + T1u;
+	       Ty = T3 - T5;
+	       T1v = T1s - T1u;
+	  }
+	  {
+	       E Td, T1A, TG, T1E, Tg, T1D, TJ, T1B;
+	       {
+		    E Tb, Tc, TE, TF;
+		    Tb = ri[WS(ris, 2)];
+		    Tc = ri[WS(ris, 14)];
+		    Td = Tb + Tc;
+		    T1A = Tb - Tc;
+		    TE = ii[WS(iis, 2)];
+		    TF = ii[WS(iis, 14)];
+		    TG = TE - TF;
+		    T1E = TE + TF;
+	       }
+	       {
+		    E Te, Tf, TH, TI;
+		    Te = ri[WS(ris, 10)];
+		    Tf = ri[WS(ris, 6)];
+		    Tg = Te + Tf;
+		    T1D = Te - Tf;
+		    TH = ii[WS(iis, 10)];
+		    TI = ii[WS(iis, 6)];
+		    TJ = TH - TI;
+		    T1B = TH + TI;
+	       }
+	       Th = KP2_000000000 * (Td + Tg);
+	       T2e = T1A + T1B;
+	       T2f = T1E - T1D;
+	       TD = Td - Tg;
+	       TK = TG - TJ;
+	       T1C = T1A - T1B;
+	       T1F = T1D + T1E;
+	       T1h = KP2_000000000 * (TJ + TG);
+	  }
+	  {
+	       E Tl, T1I, TZ, T1X, To, T1W, T12, T1J;
+	       {
+		    E Tj, Tk, TX, TY;
+		    Tj = ri[WS(ris, 1)];
+		    Tk = ri[WS(ris, 15)];
+		    Tl = Tj + Tk;
+		    T1I = Tj - Tk;
+		    TX = ii[WS(iis, 1)];
+		    TY = ii[WS(iis, 15)];
+		    TZ = TX - TY;
+		    T1X = TX + TY;
+	       }
+	       {
+		    E Tm, Tn, T10, T11;
+		    Tm = ri[WS(ris, 9)];
+		    Tn = ri[WS(ris, 7)];
+		    To = Tm + Tn;
+		    T1W = Tm - Tn;
+		    T10 = ii[WS(iis, 9)];
+		    T11 = ii[WS(iis, 7)];
+		    T12 = T10 - T11;
+		    T1J = T10 + T11;
+	       }
+	       Tp = Tl + To;
+	       T2i = T1I + T1J;
+	       T2m = T1X - T1W;
+	       TN = Tl - To;
+	       T13 = TZ - T12;
+	       T1K = T1I - T1J;
+	       T1Y = T1W + T1X;
+	       T1k = T12 + TZ;
+	  }
+	  {
+	       E Ts, T1L, TT, T1M, Tv, T1O, TQ, T1P;
+	       {
+		    E Tq, Tr, TR, TS;
+		    Tq = ri[WS(ris, 5)];
+		    Tr = ri[WS(ris, 11)];
+		    Ts = Tq + Tr;
+		    T1L = Tq - Tr;
+		    TR = ii[WS(iis, 5)];
+		    TS = ii[WS(iis, 11)];
+		    TT = TR - TS;
+		    T1M = TR + TS;
+	       }
+	       {
+		    E Tt, Tu, TO, TP;
+		    Tt = ri[WS(ris, 3)];
+		    Tu = ri[WS(ris, 13)];
+		    Tv = Tt + Tu;
+		    T1O = Tt - Tu;
+		    TO = ii[WS(iis, 13)];
+		    TP = ii[WS(iis, 3)];
+		    TQ = TO - TP;
+		    T1P = TP + TO;
+	       }
+	       Tw = Ts + Tv;
+	       TU = TQ - TT;
+	       T1l = TT + TQ;
+	       TW = Ts - Tv;
+	       {
+		    E T1T, T1U, T1N, T1Q;
+		    T1T = T1L + T1M;
+		    T1U = T1O + T1P;
+		    T1V = KP707106781 * (T1T - T1U);
+		    T2j = KP707106781 * (T1T + T1U);
+		    T1N = T1L - T1M;
+		    T1Q = T1O - T1P;
+		    T1R = KP707106781 * (T1N + T1Q);
+		    T2l = KP707106781 * (T1N - T1Q);
+	       }
+	  }
+	  {
+	       E Tx, T1r, Ti, T1q, Ta;
+	       Tx = KP2_000000000 * (Tp + Tw);
+	       T1r = KP2_000000000 * (T1l + T1k);
+	       Ta = T6 + T9;
+	       Ti = Ta + Th;
+	       T1q = Ta - Th;
+	       O[WS(os, 16)] = Ti - Tx;
+	       O[WS(os, 24)] = T1q + T1r;
+	       O[0] = Ti + Tx;
+	       O[WS(os, 8)] = T1q - T1r;
+	  }
+	  {
+	       E T1i, T1o, T1n, T1p, T1g, T1j, T1m;
+	       T1g = T6 - T9;
+	       T1i = T1g - T1h;
+	       T1o = T1g + T1h;
+	       T1j = Tp - Tw;
+	       T1m = T1k - T1l;
+	       T1n = KP1_414213562 * (T1j - T1m);
+	       T1p = KP1_414213562 * (T1j + T1m);
+	       O[WS(os, 20)] = T1i - T1n;
+	       O[WS(os, 28)] = T1o + T1p;
+	       O[WS(os, 4)] = T1i + T1n;
+	       O[WS(os, 12)] = T1o - T1p;
+	  }
+	  {
+	       E TM, T16, T15, T17;
+	       {
+		    E TC, TL, TV, T14;
+		    TC = Ty - TB;
+		    TL = KP1_414213562 * (TD - TK);
+		    TM = TC + TL;
+		    T16 = TC - TL;
+		    TV = TN + TU;
+		    T14 = TW + T13;
+		    T15 = FNMS(KP765366864, T14, KP1_847759065 * TV);
+		    T17 = FMA(KP765366864, TV, KP1_847759065 * T14);
+	       }
+	       O[WS(os, 18)] = TM - T15;
+	       O[WS(os, 26)] = T16 + T17;
+	       O[WS(os, 2)] = TM + T15;
+	       O[WS(os, 10)] = T16 - T17;
+	  }
+	  {
+	       E T2t, T2x, T2w, T2y;
+	       {
+		    E T2r, T2s, T2u, T2v;
+		    T2r = T2b + T2c;
+		    T2s = FMA(KP1_847759065, T2e, KP765366864 * T2f);
+		    T2t = T2r - T2s;
+		    T2x = T2r + T2s;
+		    T2u = T2i + T2j;
+		    T2v = T2m - T2l;
+		    T2w = FNMS(KP1_961570560, T2v, KP390180644 * T2u);
+		    T2y = FMA(KP1_961570560, T2u, KP390180644 * T2v);
+	       }
+	       O[WS(os, 23)] = T2t - T2w;
+	       O[WS(os, 31)] = T2x + T2y;
+	       O[WS(os, 7)] = T2t + T2w;
+	       O[WS(os, 15)] = T2x - T2y;
+	  }
+	  {
+	       E T1a, T1e, T1d, T1f;
+	       {
+		    E T18, T19, T1b, T1c;
+		    T18 = Ty + TB;
+		    T19 = KP1_414213562 * (TD + TK);
+		    T1a = T18 - T19;
+		    T1e = T18 + T19;
+		    T1b = TN - TU;
+		    T1c = T13 - TW;
+		    T1d = FNMS(KP1_847759065, T1c, KP765366864 * T1b);
+		    T1f = FMA(KP1_847759065, T1b, KP765366864 * T1c);
+	       }
+	       O[WS(os, 22)] = T1a - T1d;
+	       O[WS(os, 30)] = T1e + T1f;
+	       O[WS(os, 6)] = T1a + T1d;
+	       O[WS(os, 14)] = T1e - T1f;
+	  }
+	  {
+	       E T25, T29, T28, T2a;
+	       {
+		    E T23, T24, T26, T27;
+		    T23 = T1v - T1y;
+		    T24 = FMA(KP765366864, T1C, KP1_847759065 * T1F);
+		    T25 = T23 - T24;
+		    T29 = T23 + T24;
+		    T26 = T1K - T1R;
+		    T27 = T1Y - T1V;
+		    T28 = FNMS(KP1_662939224, T27, KP1_111140466 * T26);
+		    T2a = FMA(KP1_662939224, T26, KP1_111140466 * T27);
+	       }
+	       O[WS(os, 21)] = T25 - T28;
+	       O[WS(os, 29)] = T29 + T2a;
+	       O[WS(os, 5)] = T25 + T28;
+	       O[WS(os, 13)] = T29 - T2a;
+	  }
+	  {
+	       E T2h, T2p, T2o, T2q;
+	       {
+		    E T2d, T2g, T2k, T2n;
+		    T2d = T2b - T2c;
+		    T2g = FNMS(KP1_847759065, T2f, KP765366864 * T2e);
+		    T2h = T2d + T2g;
+		    T2p = T2d - T2g;
+		    T2k = T2i - T2j;
+		    T2n = T2l + T2m;
+		    T2o = FNMS(KP1_111140466, T2n, KP1_662939224 * T2k);
+		    T2q = FMA(KP1_111140466, T2k, KP1_662939224 * T2n);
+	       }
+	       O[WS(os, 19)] = T2h - T2o;
+	       O[WS(os, 27)] = T2p + T2q;
+	       O[WS(os, 3)] = T2h + T2o;
+	       O[WS(os, 11)] = T2p - T2q;
+	  }
+	  {
+	       E T1H, T21, T20, T22;
+	       {
+		    E T1z, T1G, T1S, T1Z;
+		    T1z = T1v + T1y;
+		    T1G = FNMS(KP765366864, T1F, KP1_847759065 * T1C);
+		    T1H = T1z + T1G;
+		    T21 = T1z - T1G;
+		    T1S = T1K + T1R;
+		    T1Z = T1V + T1Y;
+		    T20 = FNMS(KP390180644, T1Z, KP1_961570560 * T1S);
+		    T22 = FMA(KP390180644, T1S, KP1_961570560 * T1Z);
+	       }
+	       O[WS(os, 17)] = T1H - T20;
+	       O[WS(os, 25)] = T21 + T22;
+	       O[WS(os, 1)] = T1H + T20;
+	       O[WS(os, 9)] = T21 - T22;
+	  }
+     }
+}
+
+static void mhc2r_32(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  mhc2r_32_0(ri, ii, O, ris, iis, os);
+	  ri += ivs;
+	  ii += ivs;
+	  O += ovs;
+     }
+}
+
+static const khc2r_desc desc = { 32, "mhc2r_32", {140, 34, 16, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_mhc2r_32) (planner *p) {
+     X(khc2r_register) (p, mhc2r_32, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2r/mhc2r_64.c b/src/fftw3/rdft/codelets/hc2r/mhc2r_64.c
new file mode 100644
index 0000000..48fb66f
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2r/mhc2r_64.c
@@ -0,0 +1,733 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:11:18 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2r_noinline -compact -variables 4 -sign 1 -n 64 -name mhc2r_64 -include hc2r.h */
+
+/*
+ * This function contains 394 FP additions, 134 FP multiplications,
+ * (or, 342 additions, 82 multiplications, 52 fused multiply/add),
+ * 109 stack variables, and 128 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: mhc2r_64.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: mhc2r_64.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ * $Id: mhc2r_64.c,v 1.1 2008/10/17 06:12:08 scuri Exp $
+ */
+
+#include "hc2r.h"
+
+static void mhc2r_64_0(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os)
+{
+     DK(KP1_268786568, +1.268786568327290996430343226450986741351374190);
+     DK(KP1_546020906, +1.546020906725473921621813219516939601942082586);
+     DK(KP196034280, +0.196034280659121203988391127777283691722273346);
+     DK(KP1_990369453, +1.990369453344393772489673906218959843150949737);
+     DK(KP942793473, +0.942793473651995297112775251810508755314920638);
+     DK(KP1_763842528, +1.763842528696710059425513727320776699016885241);
+     DK(KP580569354, +0.580569354508924735272384751634790549382952557);
+     DK(KP1_913880671, +1.913880671464417729871595773960539938965698411);
+     DK(KP1_111140466, +1.111140466039204449485661627897065748749874382);
+     DK(KP1_662939224, +1.662939224605090474157576755235811513477121624);
+     DK(KP390180644, +0.390180644032256535696569736954044481855383236);
+     DK(KP1_961570560, +1.961570560806460898252364472268478073947867462);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     {
+	  E Ta, T2S, T18, T2u, T3F, T4V, T5l, T61, Th, T2T, T1h, T2v, T3M, T4W, T5o;
+	  E T62, T3Q, T5q, T5u, T44, Tp, Tw, T2V, T2W, T2X, T2Y, T3X, T5t, T1r, T2x;
+	  E T41, T5r, T1A, T2y, T4a, T5y, T5N, T4H, TN, T31, T4E, T5z, T39, T3q, T1L;
+	  E T2B, T4h, T5M, T2h, T2F, T12, T36, T5D, T5J, T5G, T5K, T1U, T26, T23, T27;
+	  E T4p, T4z, T4w, T4A, T34, T3r;
+	  {
+	       E T5, T3A, T3, T3y, T9, T3C, T17, T3D, T6, T14;
+	       {
+		    E T4, T3z, T1, T2;
+		    T4 = ri[WS(ris, 16)];
+		    T5 = KP2_000000000 * T4;
+		    T3z = ii[WS(iis, 16)];
+		    T3A = KP2_000000000 * T3z;
+		    T1 = ri[0];
+		    T2 = ri[WS(ris, 32)];
+		    T3 = T1 + T2;
+		    T3y = T1 - T2;
+		    {
+			 E T7, T8, T15, T16;
+			 T7 = ri[WS(ris, 8)];
+			 T8 = ri[WS(ris, 24)];
+			 T9 = KP2_000000000 * (T7 + T8);
+			 T3C = T7 - T8;
+			 T15 = ii[WS(iis, 8)];
+			 T16 = ii[WS(iis, 24)];
+			 T17 = KP2_000000000 * (T15 - T16);
+			 T3D = T15 + T16;
+		    }
+	       }
+	       T6 = T3 + T5;
+	       Ta = T6 + T9;
+	       T2S = T6 - T9;
+	       T14 = T3 - T5;
+	       T18 = T14 - T17;
+	       T2u = T14 + T17;
+	       {
+		    E T3B, T3E, T5j, T5k;
+		    T3B = T3y - T3A;
+		    T3E = KP1_414213562 * (T3C - T3D);
+		    T3F = T3B + T3E;
+		    T4V = T3B - T3E;
+		    T5j = T3y + T3A;
+		    T5k = KP1_414213562 * (T3C + T3D);
+		    T5l = T5j - T5k;
+		    T61 = T5j + T5k;
+	       }
+	  }
+	  {
+	       E Td, T3G, T1c, T3K, Tg, T3J, T1f, T3H, T19, T1g;
+	       {
+		    E Tb, Tc, T1a, T1b;
+		    Tb = ri[WS(ris, 4)];
+		    Tc = ri[WS(ris, 28)];
+		    Td = Tb + Tc;
+		    T3G = Tb - Tc;
+		    T1a = ii[WS(iis, 4)];
+		    T1b = ii[WS(iis, 28)];
+		    T1c = T1a - T1b;
+		    T3K = T1a + T1b;
+	       }
+	       {
+		    E Te, Tf, T1d, T1e;
+		    Te = ri[WS(ris, 20)];
+		    Tf = ri[WS(ris, 12)];
+		    Tg = Te + Tf;
+		    T3J = Te - Tf;
+		    T1d = ii[WS(iis, 20)];
+		    T1e = ii[WS(iis, 12)];
+		    T1f = T1d - T1e;
+		    T3H = T1d + T1e;
+	       }
+	       Th = KP2_000000000 * (Td + Tg);
+	       T2T = KP2_000000000 * (T1f + T1c);
+	       T19 = Td - Tg;
+	       T1g = T1c - T1f;
+	       T1h = KP1_414213562 * (T19 - T1g);
+	       T2v = KP1_414213562 * (T19 + T1g);
+	       {
+		    E T3I, T3L, T5m, T5n;
+		    T3I = T3G - T3H;
+		    T3L = T3J + T3K;
+		    T3M = FNMS(KP765366864, T3L, KP1_847759065 * T3I);
+		    T4W = FMA(KP765366864, T3I, KP1_847759065 * T3L);
+		    T5m = T3G + T3H;
+		    T5n = T3K - T3J;
+		    T5o = FNMS(KP1_847759065, T5n, KP765366864 * T5m);
+		    T62 = FMA(KP1_847759065, T5m, KP765366864 * T5n);
+	       }
+	  }
+	  {
+	       E Tl, T3O, T1v, T43, To, T42, T1y, T3P, Ts, T3R, T1p, T3S, Tv, T3U, T1m;
+	       E T3V;
+	       {
+		    E Tj, Tk, T1t, T1u;
+		    Tj = ri[WS(ris, 2)];
+		    Tk = ri[WS(ris, 30)];
+		    Tl = Tj + Tk;
+		    T3O = Tj - Tk;
+		    T1t = ii[WS(iis, 2)];
+		    T1u = ii[WS(iis, 30)];
+		    T1v = T1t - T1u;
+		    T43 = T1t + T1u;
+	       }
+	       {
+		    E Tm, Tn, T1w, T1x;
+		    Tm = ri[WS(ris, 18)];
+		    Tn = ri[WS(ris, 14)];
+		    To = Tm + Tn;
+		    T42 = Tm - Tn;
+		    T1w = ii[WS(iis, 18)];
+		    T1x = ii[WS(iis, 14)];
+		    T1y = T1w - T1x;
+		    T3P = T1w + T1x;
+	       }
+	       {
+		    E Tq, Tr, T1n, T1o;
+		    Tq = ri[WS(ris, 10)];
+		    Tr = ri[WS(ris, 22)];
+		    Ts = Tq + Tr;
+		    T3R = Tq - Tr;
+		    T1n = ii[WS(iis, 10)];
+		    T1o = ii[WS(iis, 22)];
+		    T1p = T1n - T1o;
+		    T3S = T1n + T1o;
+	       }
+	       {
+		    E Tt, Tu, T1k, T1l;
+		    Tt = ri[WS(ris, 6)];
+		    Tu = ri[WS(ris, 26)];
+		    Tv = Tt + Tu;
+		    T3U = Tt - Tu;
+		    T1k = ii[WS(iis, 26)];
+		    T1l = ii[WS(iis, 6)];
+		    T1m = T1k - T1l;
+		    T3V = T1l + T1k;
+	       }
+	       T3Q = T3O - T3P;
+	       T5q = T3O + T3P;
+	       T5u = T43 - T42;
+	       T44 = T42 + T43;
+	       Tp = Tl + To;
+	       Tw = Ts + Tv;
+	       T2V = Tp - Tw;
+	       {
+		    E T3T, T3W, T1j, T1q;
+		    T2W = T1y + T1v;
+		    T2X = T1p + T1m;
+		    T2Y = T2W - T2X;
+		    T3T = T3R - T3S;
+		    T3W = T3U - T3V;
+		    T3X = KP707106781 * (T3T + T3W);
+		    T5t = KP707106781 * (T3T - T3W);
+		    T1j = Tl - To;
+		    T1q = T1m - T1p;
+		    T1r = T1j + T1q;
+		    T2x = T1j - T1q;
+		    {
+			 E T3Z, T40, T1s, T1z;
+			 T3Z = T3R + T3S;
+			 T40 = T3U + T3V;
+			 T41 = KP707106781 * (T3Z - T40);
+			 T5r = KP707106781 * (T3Z + T40);
+			 T1s = Ts - Tv;
+			 T1z = T1v - T1y;
+			 T1A = T1s + T1z;
+			 T2y = T1z - T1s;
+		    }
+	       }
+	  }
+	  {
+	       E TB, T48, T2c, T4G, TE, T4F, T2f, T49, TI, T4b, T1J, T4c, TL, T4e, T1G;
+	       E T4f;
+	       {
+		    E Tz, TA, T2a, T2b;
+		    Tz = ri[WS(ris, 1)];
+		    TA = ri[WS(ris, 31)];
+		    TB = Tz + TA;
+		    T48 = Tz - TA;
+		    T2a = ii[WS(iis, 1)];
+		    T2b = ii[WS(iis, 31)];
+		    T2c = T2a - T2b;
+		    T4G = T2a + T2b;
+	       }
+	       {
+		    E TC, TD, T2d, T2e;
+		    TC = ri[WS(ris, 17)];
+		    TD = ri[WS(ris, 15)];
+		    TE = TC + TD;
+		    T4F = TC - TD;
+		    T2d = ii[WS(iis, 17)];
+		    T2e = ii[WS(iis, 15)];
+		    T2f = T2d - T2e;
+		    T49 = T2d + T2e;
+	       }
+	       {
+		    E TG, TH, T1H, T1I;
+		    TG = ri[WS(ris, 9)];
+		    TH = ri[WS(ris, 23)];
+		    TI = TG + TH;
+		    T4b = TG - TH;
+		    T1H = ii[WS(iis, 9)];
+		    T1I = ii[WS(iis, 23)];
+		    T1J = T1H - T1I;
+		    T4c = T1H + T1I;
+	       }
+	       {
+		    E TJ, TK, T1E, T1F;
+		    TJ = ri[WS(ris, 7)];
+		    TK = ri[WS(ris, 25)];
+		    TL = TJ + TK;
+		    T4e = TJ - TK;
+		    T1E = ii[WS(iis, 25)];
+		    T1F = ii[WS(iis, 7)];
+		    T1G = T1E - T1F;
+		    T4f = T1F + T1E;
+	       }
+	       {
+		    E TF, TM, T1D, T1K;
+		    T4a = T48 - T49;
+		    T5y = T48 + T49;
+		    T5N = T4G - T4F;
+		    T4H = T4F + T4G;
+		    TF = TB + TE;
+		    TM = TI + TL;
+		    TN = TF + TM;
+		    T31 = TF - TM;
+		    {
+			 E T4C, T4D, T37, T38;
+			 T4C = T4b + T4c;
+			 T4D = T4e + T4f;
+			 T4E = KP707106781 * (T4C - T4D);
+			 T5z = KP707106781 * (T4C + T4D);
+			 T37 = T2f + T2c;
+			 T38 = T1J + T1G;
+			 T39 = T37 - T38;
+			 T3q = T38 + T37;
+		    }
+		    T1D = TB - TE;
+		    T1K = T1G - T1J;
+		    T1L = T1D + T1K;
+		    T2B = T1D - T1K;
+		    {
+			 E T4d, T4g, T29, T2g;
+			 T4d = T4b - T4c;
+			 T4g = T4e - T4f;
+			 T4h = KP707106781 * (T4d + T4g);
+			 T5M = KP707106781 * (T4d - T4g);
+			 T29 = TI - TL;
+			 T2g = T2c - T2f;
+			 T2h = T29 + T2g;
+			 T2F = T2g - T29;
+		    }
+	       }
+	  }
+	  {
+	       E TQ, T4j, T1P, T4n, TT, T4m, T1S, T4k, TX, T4q, T1Y, T4u, T10, T4t, T21;
+	       E T4r;
+	       {
+		    E TO, TP, T1N, T1O;
+		    TO = ri[WS(ris, 5)];
+		    TP = ri[WS(ris, 27)];
+		    TQ = TO + TP;
+		    T4j = TO - TP;
+		    T1N = ii[WS(iis, 5)];
+		    T1O = ii[WS(iis, 27)];
+		    T1P = T1N - T1O;
+		    T4n = T1N + T1O;
+	       }
+	       {
+		    E TR, TS, T1Q, T1R;
+		    TR = ri[WS(ris, 21)];
+		    TS = ri[WS(ris, 11)];
+		    TT = TR + TS;
+		    T4m = TR - TS;
+		    T1Q = ii[WS(iis, 21)];
+		    T1R = ii[WS(iis, 11)];
+		    T1S = T1Q - T1R;
+		    T4k = T1Q + T1R;
+	       }
+	       {
+		    E TV, TW, T1W, T1X;
+		    TV = ri[WS(ris, 3)];
+		    TW = ri[WS(ris, 29)];
+		    TX = TV + TW;
+		    T4q = TV - TW;
+		    T1W = ii[WS(iis, 29)];
+		    T1X = ii[WS(iis, 3)];
+		    T1Y = T1W - T1X;
+		    T4u = T1X + T1W;
+	       }
+	       {
+		    E TY, TZ, T1Z, T20;
+		    TY = ri[WS(ris, 13)];
+		    TZ = ri[WS(ris, 19)];
+		    T10 = TY + TZ;
+		    T4t = TY - TZ;
+		    T1Z = ii[WS(iis, 13)];
+		    T20 = ii[WS(iis, 19)];
+		    T21 = T1Z - T20;
+		    T4r = T1Z + T20;
+	       }
+	       {
+		    E TU, T11, T5B, T5C;
+		    TU = TQ + TT;
+		    T11 = TX + T10;
+		    T12 = TU + T11;
+		    T36 = TU - T11;
+		    T5B = T4j + T4k;
+		    T5C = T4n - T4m;
+		    T5D = FNMS(KP923879532, T5C, KP382683432 * T5B);
+		    T5J = FMA(KP923879532, T5B, KP382683432 * T5C);
+	       }
+	       {
+		    E T5E, T5F, T1M, T1T;
+		    T5E = T4q + T4r;
+		    T5F = T4t + T4u;
+		    T5G = FNMS(KP923879532, T5F, KP382683432 * T5E);
+		    T5K = FMA(KP923879532, T5E, KP382683432 * T5F);
+		    T1M = TQ - TT;
+		    T1T = T1P - T1S;
+		    T1U = T1M - T1T;
+		    T26 = T1M + T1T;
+	       }
+	       {
+		    E T1V, T22, T4l, T4o;
+		    T1V = TX - T10;
+		    T22 = T1Y - T21;
+		    T23 = T1V + T22;
+		    T27 = T22 - T1V;
+		    T4l = T4j - T4k;
+		    T4o = T4m + T4n;
+		    T4p = FNMS(KP382683432, T4o, KP923879532 * T4l);
+		    T4z = FMA(KP382683432, T4l, KP923879532 * T4o);
+	       }
+	       {
+		    E T4s, T4v, T32, T33;
+		    T4s = T4q - T4r;
+		    T4v = T4t - T4u;
+		    T4w = FMA(KP923879532, T4s, KP382683432 * T4v);
+		    T4A = FNMS(KP382683432, T4s, KP923879532 * T4v);
+		    T32 = T21 + T1Y;
+		    T33 = T1S + T1P;
+		    T34 = T32 - T33;
+		    T3r = T33 + T32;
+	       }
+	  }
+	  {
+	       E T13, T3x, Ty, T3w, Ti, Tx;
+	       T13 = KP2_000000000 * (TN + T12);
+	       T3x = KP2_000000000 * (T3r + T3q);
+	       Ti = Ta + Th;
+	       Tx = KP2_000000000 * (Tp + Tw);
+	       Ty = Ti + Tx;
+	       T3w = Ti - Tx;
+	       O[WS(os, 32)] = Ty - T13;
+	       O[WS(os, 48)] = T3w + T3x;
+	       O[0] = Ty + T13;
+	       O[WS(os, 16)] = T3w - T3x;
+	  }
+	  {
+	       E T3g, T3k, T3j, T3l;
+	       {
+		    E T3e, T3f, T3h, T3i;
+		    T3e = T2S + T2T;
+		    T3f = KP1_414213562 * (T2V + T2Y);
+		    T3g = T3e - T3f;
+		    T3k = T3e + T3f;
+		    T3h = T31 - T34;
+		    T3i = T39 - T36;
+		    T3j = FNMS(KP1_847759065, T3i, KP765366864 * T3h);
+		    T3l = FMA(KP1_847759065, T3h, KP765366864 * T3i);
+	       }
+	       O[WS(os, 44)] = T3g - T3j;
+	       O[WS(os, 60)] = T3k + T3l;
+	       O[WS(os, 12)] = T3g + T3j;
+	       O[WS(os, 28)] = T3k - T3l;
+	  }
+	  {
+	       E T3o, T3u, T3t, T3v;
+	       {
+		    E T3m, T3n, T3p, T3s;
+		    T3m = Ta - Th;
+		    T3n = KP2_000000000 * (T2X + T2W);
+		    T3o = T3m - T3n;
+		    T3u = T3m + T3n;
+		    T3p = TN - T12;
+		    T3s = T3q - T3r;
+		    T3t = KP1_414213562 * (T3p - T3s);
+		    T3v = KP1_414213562 * (T3p + T3s);
+	       }
+	       O[WS(os, 40)] = T3o - T3t;
+	       O[WS(os, 56)] = T3u + T3v;
+	       O[WS(os, 8)] = T3o + T3t;
+	       O[WS(os, 24)] = T3u - T3v;
+	  }
+	  {
+	       E T30, T3c, T3b, T3d;
+	       {
+		    E T2U, T2Z, T35, T3a;
+		    T2U = T2S - T2T;
+		    T2Z = KP1_414213562 * (T2V - T2Y);
+		    T30 = T2U + T2Z;
+		    T3c = T2U - T2Z;
+		    T35 = T31 + T34;
+		    T3a = T36 + T39;
+		    T3b = FNMS(KP765366864, T3a, KP1_847759065 * T35);
+		    T3d = FMA(KP765366864, T35, KP1_847759065 * T3a);
+	       }
+	       O[WS(os, 36)] = T30 - T3b;
+	       O[WS(os, 52)] = T3c + T3d;
+	       O[WS(os, 4)] = T30 + T3b;
+	       O[WS(os, 20)] = T3c - T3d;
+	  }
+	  {
+	       E T25, T2p, T2i, T2q, T1C, T2k, T2o, T2s, T24, T28;
+	       T24 = KP707106781 * (T1U + T23);
+	       T25 = T1L + T24;
+	       T2p = T1L - T24;
+	       T28 = KP707106781 * (T26 + T27);
+	       T2i = T28 + T2h;
+	       T2q = T2h - T28;
+	       {
+		    E T1i, T1B, T2m, T2n;
+		    T1i = T18 + T1h;
+		    T1B = FNMS(KP765366864, T1A, KP1_847759065 * T1r);
+		    T1C = T1i + T1B;
+		    T2k = T1i - T1B;
+		    T2m = T18 - T1h;
+		    T2n = FMA(KP765366864, T1r, KP1_847759065 * T1A);
+		    T2o = T2m - T2n;
+		    T2s = T2m + T2n;
+	       }
+	       {
+		    E T2j, T2t, T2l, T2r;
+		    T2j = FNMS(KP390180644, T2i, KP1_961570560 * T25);
+		    O[WS(os, 34)] = T1C - T2j;
+		    O[WS(os, 2)] = T1C + T2j;
+		    T2t = FMA(KP1_662939224, T2p, KP1_111140466 * T2q);
+		    O[WS(os, 26)] = T2s - T2t;
+		    O[WS(os, 58)] = T2s + T2t;
+		    T2l = FMA(KP390180644, T25, KP1_961570560 * T2i);
+		    O[WS(os, 18)] = T2k - T2l;
+		    O[WS(os, 50)] = T2k + T2l;
+		    T2r = FNMS(KP1_662939224, T2q, KP1_111140466 * T2p);
+		    O[WS(os, 42)] = T2o - T2r;
+		    O[WS(os, 10)] = T2o + T2r;
+	       }
+	  }
+	  {
+	       E T2D, T2N, T2G, T2O, T2A, T2I, T2M, T2Q, T2C, T2E;
+	       T2C = KP707106781 * (T27 - T26);
+	       T2D = T2B + T2C;
+	       T2N = T2B - T2C;
+	       T2E = KP707106781 * (T1U - T23);
+	       T2G = T2E + T2F;
+	       T2O = T2F - T2E;
+	       {
+		    E T2w, T2z, T2K, T2L;
+		    T2w = T2u - T2v;
+		    T2z = FNMS(KP1_847759065, T2y, KP765366864 * T2x);
+		    T2A = T2w + T2z;
+		    T2I = T2w - T2z;
+		    T2K = T2u + T2v;
+		    T2L = FMA(KP1_847759065, T2x, KP765366864 * T2y);
+		    T2M = T2K - T2L;
+		    T2Q = T2K + T2L;
+	       }
+	       {
+		    E T2H, T2R, T2J, T2P;
+		    T2H = FNMS(KP1_111140466, T2G, KP1_662939224 * T2D);
+		    O[WS(os, 38)] = T2A - T2H;
+		    O[WS(os, 6)] = T2A + T2H;
+		    T2R = FMA(KP1_961570560, T2N, KP390180644 * T2O);
+		    O[WS(os, 30)] = T2Q - T2R;
+		    O[WS(os, 62)] = T2Q + T2R;
+		    T2J = FMA(KP1_111140466, T2D, KP1_662939224 * T2G);
+		    O[WS(os, 22)] = T2I - T2J;
+		    O[WS(os, 54)] = T2I + T2J;
+		    T2P = FNMS(KP1_961570560, T2O, KP390180644 * T2N);
+		    O[WS(os, 46)] = T2M - T2P;
+		    O[WS(os, 14)] = T2M + T2P;
+	       }
+	  }
+	  {
+	       E T5p, T5T, T5w, T5U, T5I, T5W, T5P, T5X, T5s, T5v;
+	       T5p = T5l + T5o;
+	       T5T = T5l - T5o;
+	       T5s = T5q - T5r;
+	       T5v = T5t + T5u;
+	       T5w = FNMS(KP1_111140466, T5v, KP1_662939224 * T5s);
+	       T5U = FMA(KP1_111140466, T5s, KP1_662939224 * T5v);
+	       {
+		    E T5A, T5H, T5L, T5O;
+		    T5A = T5y - T5z;
+		    T5H = T5D + T5G;
+		    T5I = T5A + T5H;
+		    T5W = T5A - T5H;
+		    T5L = T5J - T5K;
+		    T5O = T5M + T5N;
+		    T5P = T5L + T5O;
+		    T5X = T5O - T5L;
+	       }
+	       {
+		    E T5x, T5Q, T5Z, T60;
+		    T5x = T5p + T5w;
+		    T5Q = FNMS(KP580569354, T5P, KP1_913880671 * T5I);
+		    O[WS(os, 35)] = T5x - T5Q;
+		    O[WS(os, 3)] = T5x + T5Q;
+		    T5Z = T5T + T5U;
+		    T60 = FMA(KP1_763842528, T5W, KP942793473 * T5X);
+		    O[WS(os, 27)] = T5Z - T60;
+		    O[WS(os, 59)] = T5Z + T60;
+	       }
+	       {
+		    E T5R, T5S, T5V, T5Y;
+		    T5R = T5p - T5w;
+		    T5S = FMA(KP580569354, T5I, KP1_913880671 * T5P);
+		    O[WS(os, 19)] = T5R - T5S;
+		    O[WS(os, 51)] = T5R + T5S;
+		    T5V = T5T - T5U;
+		    T5Y = FNMS(KP1_763842528, T5X, KP942793473 * T5W);
+		    O[WS(os, 43)] = T5V - T5Y;
+		    O[WS(os, 11)] = T5V + T5Y;
+	       }
+	  }
+	  {
+	       E T3N, T4N, T46, T4O, T4y, T4Q, T4J, T4R, T3Y, T45;
+	       T3N = T3F + T3M;
+	       T4N = T3F - T3M;
+	       T3Y = T3Q + T3X;
+	       T45 = T41 + T44;
+	       T46 = FNMS(KP390180644, T45, KP1_961570560 * T3Y);
+	       T4O = FMA(KP390180644, T3Y, KP1_961570560 * T45);
+	       {
+		    E T4i, T4x, T4B, T4I;
+		    T4i = T4a + T4h;
+		    T4x = T4p + T4w;
+		    T4y = T4i + T4x;
+		    T4Q = T4i - T4x;
+		    T4B = T4z + T4A;
+		    T4I = T4E + T4H;
+		    T4J = T4B + T4I;
+		    T4R = T4I - T4B;
+	       }
+	       {
+		    E T47, T4K, T4T, T4U;
+		    T47 = T3N + T46;
+		    T4K = FNMS(KP196034280, T4J, KP1_990369453 * T4y);
+		    O[WS(os, 33)] = T47 - T4K;
+		    O[WS(os, 1)] = T47 + T4K;
+		    T4T = T4N + T4O;
+		    T4U = FMA(KP1_546020906, T4Q, KP1_268786568 * T4R);
+		    O[WS(os, 25)] = T4T - T4U;
+		    O[WS(os, 57)] = T4T + T4U;
+	       }
+	       {
+		    E T4L, T4M, T4P, T4S;
+		    T4L = T3N - T46;
+		    T4M = FMA(KP196034280, T4y, KP1_990369453 * T4J);
+		    O[WS(os, 17)] = T4L - T4M;
+		    O[WS(os, 49)] = T4L + T4M;
+		    T4P = T4N - T4O;
+		    T4S = FNMS(KP1_546020906, T4R, KP1_268786568 * T4Q);
+		    O[WS(os, 41)] = T4P - T4S;
+		    O[WS(os, 9)] = T4P + T4S;
+	       }
+	  }
+	  {
+	       E T63, T6h, T66, T6i, T6a, T6k, T6d, T6l, T64, T65;
+	       T63 = T61 - T62;
+	       T6h = T61 + T62;
+	       T64 = T5q + T5r;
+	       T65 = T5u - T5t;
+	       T66 = FNMS(KP1_961570560, T65, KP390180644 * T64);
+	       T6i = FMA(KP1_961570560, T64, KP390180644 * T65);
+	       {
+		    E T68, T69, T6b, T6c;
+		    T68 = T5y + T5z;
+		    T69 = T5J + T5K;
+		    T6a = T68 - T69;
+		    T6k = T68 + T69;
+		    T6b = T5D - T5G;
+		    T6c = T5N - T5M;
+		    T6d = T6b + T6c;
+		    T6l = T6c - T6b;
+	       }
+	       {
+		    E T67, T6e, T6n, T6o;
+		    T67 = T63 + T66;
+		    T6e = FNMS(KP1_268786568, T6d, KP1_546020906 * T6a);
+		    O[WS(os, 39)] = T67 - T6e;
+		    O[WS(os, 7)] = T67 + T6e;
+		    T6n = T6h + T6i;
+		    T6o = FMA(KP1_990369453, T6k, KP196034280 * T6l);
+		    O[WS(os, 31)] = T6n - T6o;
+		    O[WS(os, 63)] = T6n + T6o;
+	       }
+	       {
+		    E T6f, T6g, T6j, T6m;
+		    T6f = T63 - T66;
+		    T6g = FMA(KP1_268786568, T6a, KP1_546020906 * T6d);
+		    O[WS(os, 23)] = T6f - T6g;
+		    O[WS(os, 55)] = T6f + T6g;
+		    T6j = T6h - T6i;
+		    T6m = FNMS(KP1_990369453, T6l, KP196034280 * T6k);
+		    O[WS(os, 47)] = T6j - T6m;
+		    O[WS(os, 15)] = T6j + T6m;
+	       }
+	  }
+	  {
+	       E T4X, T5b, T50, T5c, T54, T5e, T57, T5f, T4Y, T4Z;
+	       T4X = T4V - T4W;
+	       T5b = T4V + T4W;
+	       T4Y = T3Q - T3X;
+	       T4Z = T44 - T41;
+	       T50 = FNMS(KP1_662939224, T4Z, KP1_111140466 * T4Y);
+	       T5c = FMA(KP1_662939224, T4Y, KP1_111140466 * T4Z);
+	       {
+		    E T52, T53, T55, T56;
+		    T52 = T4a - T4h;
+		    T53 = T4A - T4z;
+		    T54 = T52 + T53;
+		    T5e = T52 - T53;
+		    T55 = T4p - T4w;
+		    T56 = T4H - T4E;
+		    T57 = T55 + T56;
+		    T5f = T56 - T55;
+	       }
+	       {
+		    E T51, T58, T5h, T5i;
+		    T51 = T4X + T50;
+		    T58 = FNMS(KP942793473, T57, KP1_763842528 * T54);
+		    O[WS(os, 37)] = T51 - T58;
+		    O[WS(os, 5)] = T51 + T58;
+		    T5h = T5b + T5c;
+		    T5i = FMA(KP1_913880671, T5e, KP580569354 * T5f);
+		    O[WS(os, 29)] = T5h - T5i;
+		    O[WS(os, 61)] = T5h + T5i;
+	       }
+	       {
+		    E T59, T5a, T5d, T5g;
+		    T59 = T4X - T50;
+		    T5a = FMA(KP942793473, T54, KP1_763842528 * T57);
+		    O[WS(os, 21)] = T59 - T5a;
+		    O[WS(os, 53)] = T59 + T5a;
+		    T5d = T5b - T5c;
+		    T5g = FNMS(KP1_913880671, T5f, KP580569354 * T5e);
+		    O[WS(os, 45)] = T5d - T5g;
+		    O[WS(os, 13)] = T5d + T5g;
+	       }
+	  }
+     }
+}
+
+static void mhc2r_64(const R *ri, const R *ii, R *O, stride ris, stride iis, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  mhc2r_64_0(ri, ii, O, ris, iis, os);
+	  ri += ivs;
+	  ii += ivs;
+	  O += ovs;
+     }
+}
+
+static const khc2r_desc desc = { 64, "mhc2r_64", {342, 82, 52, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_mhc2r_64) (planner *p) {
+     X(khc2r_register) (p, mhc2r_64, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/hc2rIII.h b/src/fftw3/rdft/codelets/hc2rIII.h
new file mode 100644
index 0000000..f14d248
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hc2rIII.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+
+#define GENUS X(rdft_hc2rIII_genus)
+extern const khc2rIII_genus GENUS;
diff --git a/src/fftw3/rdft/codelets/hf.h b/src/fftw3/rdft/codelets/hf.h
new file mode 100644
index 0000000..ff80d36
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hf.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+
+#define GENUS X(rdft_hf_genus)
+extern const hc2hc_genus GENUS;
diff --git a/src/fftw3/rdft/codelets/hfb.c b/src/fftw3/rdft/codelets/hfb.c
new file mode 100644
index 0000000..93b2bb6
--- /dev/null
+++ b/src/fftw3/rdft/codelets/hfb.c
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "codelet-rdft.h"
+#include "hf.h"
+
+static int okp(const hc2hc_desc *d,
+	       const R *rio, const R *iio, 
+	       int ios, int vs, int m, int dist)
+{
+     UNUSED(rio); UNUSED(iio); UNUSED(m);
+     return (1
+	     && (!d->s1 || (d->s1 == ios))
+	     && (!d->s2 || (d->s2 == vs))
+	     && (!d->dist || (d->dist == dist))
+	  );
+}
+
+const hc2hc_genus GENUS = { okp, R2HC, 1 };
+
+#undef GENUS
+#include "hb.h"
+
+const hc2hc_genus GENUS = { okp, HC2R, 1 };
diff --git a/src/fftw3/rdft/codelets/r2hc.c b/src/fftw3/rdft/codelets/r2hc.c
new file mode 100644
index 0000000..1506868
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "codelet-rdft.h"
+#include "r2hc.h"
+
+static int okp(const kr2hc_desc *d,
+	       const R *I,
+	       const R *ro, const R *io,
+	       int is, int ros, int ios, int vl, int ivs, int ovs)
+{
+     UNUSED(I); UNUSED(ro); UNUSED(io); UNUSED(vl);
+     return (1
+	     && (!d->is || (d->is == is))
+	     && (!d->ros || (d->ros == ros))
+	     && (!d->ios || (d->ros == ios))
+	     && (!d->ivs || (d->ivs == ivs))
+	     && (!d->ovs || (d->ovs == ovs))
+	  );
+}
+
+const kr2hc_genus GENUS = { okp, R2HC, 1 };
+
+#undef GENUS
+#include "r2hcII.h"
+
+const kr2hcII_genus GENUS = { okp, R2HCII, 1 };
diff --git a/src/fftw3/rdft/codelets/r2hc.h b/src/fftw3/rdft/codelets/r2hc.h
new file mode 100644
index 0000000..6ec1e28
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+
+#define GENUS X(rdft_r2hc_genus)
+extern const kr2hc_genus GENUS;
diff --git a/src/fftw3/rdft/codelets/r2hc/hf2_16.c b/src/fftw3/rdft/codelets/r2hc/hf2_16.c
new file mode 100644
index 0000000..47323ee
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf2_16.c
@@ -0,0 +1,411 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:30 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -twiddle-log3 -n 16 -dit -name hf2_16 -include hf.h */
+
+/*
+ * This function contains 196 FP additions, 108 FP multiplications,
+ * (or, 156 additions, 68 multiplications, 40 fused multiply/add),
+ * 104 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf2_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf2_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf2_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf2_16(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 8) {
+	  E T1, T3d, T18, T26, T29, T2R, Tq, T1r, T1E, T2k, T2g, T1O, Te, T3c, Tz;
+	  E T1P, T1S, T1T, T1U, TG, TL, T1V, T1Y, T1Z, T20, TT, TY, T1X, T1A, T2l;
+	  E T1J, T2h, T1h, T2b, T1m, T2a;
+	  T1 = rio[0];
+	  T3d = iio[-WS(ios, 15)];
+	  {
+	       E T9, T1z, Td, T1v, T1I, Tl, Tp, T1G, Tu, T1D, TD, Ty, T1C, T1l, TX;
+	       E TK, T1g, TI, T1j, TF, T1c, TQ, TS, T1p, T1q, TV, T2, T5, Ti, Tg;
+	       E T4, Tw, Ts, Ta, Tv, T7, Tb, Tr, Tk, TW, TJ, TC, TU, To, TE;
+	       E TH, T14, T24, T17, T25, TN, TO, TP, TR;
+	       T9 = rio[WS(ios, 8)];
+	       T1z = iio[-WS(ios, 8)];
+	       Td = iio[-WS(ios, 7)];
+	       T1v = rio[WS(ios, 7)];
+	       T1I = iio[-WS(ios, 4)];
+	       Tl = rio[WS(ios, 4)];
+	       Tp = iio[-WS(ios, 11)];
+	       T1G = rio[WS(ios, 11)];
+	       Tu = rio[WS(ios, 12)];
+	       T1D = iio[-WS(ios, 12)];
+	       TD = rio[WS(ios, 2)];
+	       Ty = iio[-WS(ios, 3)];
+	       T1C = rio[WS(ios, 3)];
+	       T1l = iio[-WS(ios, 2)];
+	       TX = iio[-WS(ios, 9)];
+	       TK = iio[-WS(ios, 5)];
+	       T1g = iio[-WS(ios, 10)];
+	       TI = rio[WS(ios, 10)];
+	       T1j = rio[WS(ios, 13)];
+	       TF = iio[-WS(ios, 13)];
+	       T1c = rio[WS(ios, 5)];
+	       TQ = rio[WS(ios, 14)];
+	       TS = iio[-WS(ios, 1)];
+	       T1p = rio[WS(ios, 15)];
+	       T1q = iio[0];
+	       TV = rio[WS(ios, 6)];
+	       {
+		    E T12, T16, T13, T15, T3, T6, Tm, Tj, Tn, Th;
+		    T12 = rio[WS(ios, 1)];
+		    T16 = iio[-WS(ios, 6)];
+		    T13 = iio[-WS(ios, 14)];
+		    T15 = rio[WS(ios, 9)];
+		    T2 = W[4];
+		    T5 = W[5];
+		    T3 = W[0];
+		    T6 = W[1];
+		    Ti = W[3];
+		    Tg = W[2];
+		    T4 = T2 * T3;
+		    Tw = T5 * Tg;
+		    Ts = T5 * Ti;
+		    Ta = T2 * T6;
+		    Tv = T2 * Ti;
+		    T7 = T5 * T6;
+		    Tb = T5 * T3;
+		    Tr = T2 * Tg;
+		    Tm = Tg * T6;
+		    Tj = Ti * T6;
+		    Tn = Ti * T3;
+		    Th = Tg * T3;
+		    Tk = Th - Tj;
+		    TW = Tv - Tw;
+		    TJ = Ta + Tb;
+		    TC = Th + Tj;
+		    TU = Tr + Ts;
+		    To = Tm + Tn;
+		    TE = Tm - Tn;
+		    TH = T4 - T7;
+		    T14 = FMA(T3, T12, T6 * T13);
+		    T24 = FNMS(T6, T12, T3 * T13);
+		    T17 = FMA(T2, T15, T5 * T16);
+		    T25 = FNMS(T5, T15, T2 * T16);
+		    TN = W[6];
+		    TO = W[7];
+		    TP = FMA(TN, T3, TO * T6);
+		    TR = FNMS(TO, T3, TN * T6);
+	       }
+	       T18 = T14 + T17;
+	       T26 = T24 - T25;
+	       T29 = T14 - T17;
+	       T2R = T24 + T25;
+	       Tq = FMA(Tk, Tl, To * Tp);
+	       T1r = FMA(TN, T1p, TO * T1q);
+	       T1E = FMA(Tg, T1C, Ti * T1D);
+	       T2k = FNMS(TO, T1p, TN * T1q);
+	       T2g = FNMS(Ti, T1C, Tg * T1D);
+	       {
+		    E T8, Tc, Tt, Tx;
+		    T1O = FNMS(To, Tl, Tk * Tp);
+		    T8 = T4 + T7;
+		    Tc = Ta - Tb;
+		    Te = FNMS(Tc, Td, T8 * T9);
+		    T3c = FMA(Tc, T9, T8 * Td);
+		    Tt = Tr - Ts;
+		    Tx = Tv + Tw;
+		    Tz = FMA(Tt, Tu, Tx * Ty);
+		    T1P = FNMS(Tx, Tu, Tt * Ty);
+		    T1S = FMA(TE, TD, TC * TF);
+		    T1T = FNMS(TJ, TI, TH * TK);
+		    T1U = T1S - T1T;
+	       }
+	       TG = FNMS(TE, TF, TC * TD);
+	       TL = FMA(TH, TI, TJ * TK);
+	       T1V = TG - TL;
+	       T1Y = FMA(TR, TQ, TP * TS);
+	       T1Z = FMA(TW, TV, TU * TX);
+	       T20 = T1Y - T1Z;
+	       TT = FNMS(TR, TS, TP * TQ);
+	       TY = FNMS(TW, TX, TU * TV);
+	       T1X = TT - TY;
+	       {
+		    E T1u, T1F, T1y, T1H;
+		    {
+			 E T1s, T1t, T1w, T1x;
+			 T1s = T2 * TC;
+			 T1t = T5 * TE;
+			 T1u = T1s - T1t;
+			 T1F = T1s + T1t;
+			 T1w = T2 * TE;
+			 T1x = T5 * TC;
+			 T1y = T1w + T1x;
+			 T1H = T1w - T1x;
+		    }
+		    T1A = FMA(T1u, T1v, T1y * T1z);
+		    T2l = FNMS(T1y, T1v, T1u * T1z);
+		    T1J = FNMS(T1H, T1I, T1F * T1G);
+		    T2h = FMA(T1H, T1G, T1F * T1I);
+	       }
+	       {
+		    E T1b, T1i, T1f, T1k;
+		    {
+			 E T19, T1a, T1d, T1e;
+			 T19 = T2 * Tk;
+			 T1a = T5 * To;
+			 T1b = T19 + T1a;
+			 T1i = T19 - T1a;
+			 T1d = T2 * To;
+			 T1e = T5 * Tk;
+			 T1f = T1d - T1e;
+			 T1k = T1d + T1e;
+		    }
+		    T1h = FNMS(T1f, T1g, T1b * T1c);
+		    T2b = FNMS(T1k, T1j, T1i * T1l);
+		    T1m = FMA(T1i, T1j, T1k * T1l);
+		    T2a = FMA(T1f, T1c, T1b * T1g);
+	       }
+	  }
+	  {
+	       E TB, T2L, T10, T3k, T3f, T3l, T2O, T3a, T1o, T36, T2U, T32, T1L, T37, T2Z;
+	       E T33;
+	       {
+		    E Tf, TA, T2M, T2N;
+		    Tf = T1 + Te;
+		    TA = Tq + Tz;
+		    TB = Tf + TA;
+		    T2L = Tf - TA;
+		    {
+			 E TM, TZ, T3b, T3e;
+			 TM = TG + TL;
+			 TZ = TT + TY;
+			 T10 = TM + TZ;
+			 T3k = TZ - TM;
+			 T3b = T1O + T1P;
+			 T3e = T3c + T3d;
+			 T3f = T3b + T3e;
+			 T3l = T3e - T3b;
+		    }
+		    T2M = T1S + T1T;
+		    T2N = T1Y + T1Z;
+		    T2O = T2M - T2N;
+		    T3a = T2M + T2N;
+		    {
+			 E T1n, T2Q, T2S, T2T;
+			 T1n = T1h + T1m;
+			 T2Q = T18 - T1n;
+			 T2S = T2a + T2b;
+			 T2T = T2R - T2S;
+			 T1o = T18 + T1n;
+			 T36 = T2R + T2S;
+			 T2U = T2Q + T2T;
+			 T32 = T2T - T2Q;
+		    }
+		    {
+			 E T1B, T1K, T2V, T2W, T2X, T2Y;
+			 T1B = T1r + T1A;
+			 T1K = T1E + T1J;
+			 T2V = T1B - T1K;
+			 T2W = T2k + T2l;
+			 T2X = T2g + T2h;
+			 T2Y = T2W - T2X;
+			 T1L = T1B + T1K;
+			 T37 = T2W + T2X;
+			 T2Z = T2V - T2Y;
+			 T33 = T2V + T2Y;
+		    }
+	       }
+	       {
+		    E T11, T1M, T39, T3g;
+		    T11 = TB + T10;
+		    T1M = T1o + T1L;
+		    iio[-WS(ios, 8)] = T11 - T1M;
+		    rio[0] = T11 + T1M;
+		    T39 = T36 + T37;
+		    T3g = T3a + T3f;
+		    rio[WS(ios, 8)] = T39 - T3g;
+		    iio[0] = T39 + T3g;
+	       }
+	       {
+		    E T2P, T30, T3j, T3m;
+		    T2P = T2L + T2O;
+		    T30 = KP707106781 * (T2U + T2Z);
+		    iio[-WS(ios, 10)] = T2P - T30;
+		    rio[WS(ios, 2)] = T2P + T30;
+		    T3j = KP707106781 * (T32 + T33);
+		    T3m = T3k + T3l;
+		    rio[WS(ios, 10)] = T3j - T3m;
+		    iio[-WS(ios, 2)] = T3j + T3m;
+	       }
+	       {
+		    E T31, T34, T3n, T3o;
+		    T31 = T2L - T2O;
+		    T34 = KP707106781 * (T32 - T33);
+		    iio[-WS(ios, 14)] = T31 - T34;
+		    rio[WS(ios, 6)] = T31 + T34;
+		    T3n = KP707106781 * (T2Z - T2U);
+		    T3o = T3l - T3k;
+		    rio[WS(ios, 14)] = T3n - T3o;
+		    iio[-WS(ios, 6)] = T3n + T3o;
+	       }
+	       {
+		    E T35, T38, T3h, T3i;
+		    T35 = TB - T10;
+		    T38 = T36 - T37;
+		    iio[-WS(ios, 12)] = T35 - T38;
+		    rio[WS(ios, 4)] = T35 + T38;
+		    T3h = T1L - T1o;
+		    T3i = T3f - T3a;
+		    rio[WS(ios, 12)] = T3h - T3i;
+		    iio[-WS(ios, 4)] = T3h + T3i;
+	       }
+	  }
+	  {
+	       E T1R, T2v, T22, T3q, T3t, T3z, T2y, T3y, T2e, T2I, T2s, T2C, T2p, T2J, T2t;
+	       E T2F;
+	       {
+		    E T1N, T1Q, T2w, T2x;
+		    T1N = T1 - Te;
+		    T1Q = T1O - T1P;
+		    T1R = T1N - T1Q;
+		    T2v = T1N + T1Q;
+		    {
+			 E T1W, T21, T3r, T3s;
+			 T1W = T1U - T1V;
+			 T21 = T1X + T20;
+			 T22 = KP707106781 * (T1W - T21);
+			 T3q = KP707106781 * (T1W + T21);
+			 T3r = T3d - T3c;
+			 T3s = Tq - Tz;
+			 T3t = T3r - T3s;
+			 T3z = T3s + T3r;
+		    }
+		    T2w = T1V + T1U;
+		    T2x = T1X - T20;
+		    T2y = KP707106781 * (T2w + T2x);
+		    T3y = KP707106781 * (T2x - T2w);
+		    {
+			 E T28, T2A, T2d, T2B, T27, T2c;
+			 T27 = T1h - T1m;
+			 T28 = T26 + T27;
+			 T2A = T26 - T27;
+			 T2c = T2a - T2b;
+			 T2d = T29 - T2c;
+			 T2B = T29 + T2c;
+			 T2e = FMA(KP923879532, T28, KP382683432 * T2d);
+			 T2I = FNMS(KP382683432, T2B, KP923879532 * T2A);
+			 T2s = FNMS(KP923879532, T2d, KP382683432 * T28);
+			 T2C = FMA(KP382683432, T2A, KP923879532 * T2B);
+		    }
+		    {
+			 E T2j, T2D, T2o, T2E;
+			 {
+			      E T2f, T2i, T2m, T2n;
+			      T2f = T1r - T1A;
+			      T2i = T2g - T2h;
+			      T2j = T2f - T2i;
+			      T2D = T2f + T2i;
+			      T2m = T2k - T2l;
+			      T2n = T1E - T1J;
+			      T2o = T2m + T2n;
+			      T2E = T2m - T2n;
+			 }
+			 T2p = FNMS(KP923879532, T2o, KP382683432 * T2j);
+			 T2J = FMA(KP923879532, T2E, KP382683432 * T2D);
+			 T2t = FMA(KP382683432, T2o, KP923879532 * T2j);
+			 T2F = FNMS(KP382683432, T2E, KP923879532 * T2D);
+		    }
+	       }
+	       {
+		    E T23, T2q, T3x, T3A;
+		    T23 = T1R + T22;
+		    T2q = T2e + T2p;
+		    iio[-WS(ios, 11)] = T23 - T2q;
+		    rio[WS(ios, 3)] = T23 + T2q;
+		    T3x = T2s + T2t;
+		    T3A = T3y + T3z;
+		    rio[WS(ios, 11)] = T3x - T3A;
+		    iio[-WS(ios, 3)] = T3x + T3A;
+	       }
+	       {
+		    E T2r, T2u, T3B, T3C;
+		    T2r = T1R - T22;
+		    T2u = T2s - T2t;
+		    iio[-WS(ios, 15)] = T2r - T2u;
+		    rio[WS(ios, 7)] = T2r + T2u;
+		    T3B = T2p - T2e;
+		    T3C = T3z - T3y;
+		    rio[WS(ios, 15)] = T3B - T3C;
+		    iio[-WS(ios, 7)] = T3B + T3C;
+	       }
+	       {
+		    E T2z, T2G, T3p, T3u;
+		    T2z = T2v + T2y;
+		    T2G = T2C + T2F;
+		    iio[-WS(ios, 9)] = T2z - T2G;
+		    rio[WS(ios, 1)] = T2z + T2G;
+		    T3p = T2I + T2J;
+		    T3u = T3q + T3t;
+		    rio[WS(ios, 9)] = T3p - T3u;
+		    iio[-WS(ios, 1)] = T3p + T3u;
+	       }
+	       {
+		    E T2H, T2K, T3v, T3w;
+		    T2H = T2v - T2y;
+		    T2K = T2I - T2J;
+		    iio[-WS(ios, 13)] = T2H - T2K;
+		    rio[WS(ios, 5)] = T2H + T2K;
+		    T3v = T2F - T2C;
+		    T3w = T3t - T3q;
+		    rio[WS(ios, 13)] = T3v - T3w;
+		    iio[-WS(ios, 5)] = T3v + T3w;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_COS, 0, 1},
+     {TW_SIN, 0, 1},
+     {TW_COS, 0, 3},
+     {TW_SIN, 0, 3},
+     {TW_COS, 0, 9},
+     {TW_SIN, 0, 9},
+     {TW_COS, 0, 15},
+     {TW_SIN, 0, 15},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 16, "hf2_16", twinstr, {156, 68, 40, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf2_16) (planner *p) {
+     X(khc2hc_dit_register) (p, hf2_16, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf2_32.c b/src/fftw3/rdft/codelets/r2hc/hf2_32.c
new file mode 100644
index 0000000..56c7769
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf2_32.c
@@ -0,0 +1,853 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:34 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -twiddle-log3 -n 32 -dit -name hf2_32 -include hf.h */
+
+/*
+ * This function contains 488 FP additions, 280 FP multiplications,
+ * (or, 376 additions, 168 multiplications, 112 fused multiply/add),
+ * 204 stack variables, and 128 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf2_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf2_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf2_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf2_32(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 8) {
+	  E T1, T3t, T4S, TQ, T3G, T49, T20, T2n, T4y, T1J, T43, T2w, T4z, T36, T4Z;
+	  E TK, T8b, T40, T6l, T3U, T6k, T1h, T3L, T1D, T3V, T1s, T3X, T3E, T7E, T3O;
+	  E T6h, T2k, T6w, T4i, T4x, T3q, T6I, T4O, T4P, T3w, T4T, T4R, T4U, Tm, To;
+	  E TX, T4I, T3a, T3H, T31, T4Y, T3f, T4J, T2G, T4s, T4r, T2B, T4q, T4t, T27;
+	  E T4a, T2M, T4m, T4n, T2P, T4l, T4o, T1U, T44, Tn, Tp, T7G;
+	  T1 = rio[0];
+	  {
+	       E Tv, T3e, Tz, T3c, T39, TE, TI, T38, TN, T3v, TU, T1l, T3u, TW, T12;
+	       E T35, T1f, T3m, T1b, T3o, T30, T2L, T16, T1k, T33, T2W, TP, T2J, T2D, T1w;
+	       E T1z, T2A, T1B, T2z, T2F, T1u, T2N, T1q, T1I, T2O, T1o, T1G, T2m, T24, T1Z;
+	       E T2r, T1X, T2v, T2l, T26, T2i, T1R, T1T, T2g, T2, Ti, T3, Tc, TF, TC;
+	       E TG, TB, Tu, T1a, T15, Ty, T1t, T1Y, T1W, T1v, TH, T1y, T11, TD, T1A;
+	       E T1e, T4g, T3k, T1n, T1p, T2e, T4M, TM, T1K, T1O, TO, T1L, T1N, Ta, Tb;
+	       E T2t, Tk, T2o, Tf, Tg, T2s, Tj, T2p;
+	       Tv = rio[WS(ios, 8)];
+	       T3e = iio[-WS(ios, 8)];
+	       Tz = iio[-WS(ios, 23)];
+	       T3c = rio[WS(ios, 23)];
+	       T39 = iio[-WS(ios, 24)];
+	       TE = rio[WS(ios, 24)];
+	       TI = iio[-WS(ios, 7)];
+	       T38 = rio[WS(ios, 7)];
+	       TN = rio[WS(ios, 4)];
+	       T3v = iio[-WS(ios, 20)];
+	       TU = rio[WS(ios, 20)];
+	       T1l = iio[-WS(ios, 29)];
+	       T3u = rio[WS(ios, 11)];
+	       TW = iio[-WS(ios, 11)];
+	       T12 = rio[WS(ios, 28)];
+	       T35 = iio[-WS(ios, 16)];
+	       T1f = iio[-WS(ios, 19)];
+	       T3m = rio[WS(ios, 19)];
+	       T1b = rio[WS(ios, 12)];
+	       T3o = iio[-WS(ios, 12)];
+	       T30 = iio[0];
+	       T2L = iio[-WS(ios, 2)];
+	       T16 = iio[-WS(ios, 3)];
+	       T1k = rio[WS(ios, 2)];
+	       T33 = rio[WS(ios, 15)];
+	       T2W = rio[WS(ios, 31)];
+	       TP = iio[-WS(ios, 27)];
+	       T2J = rio[WS(ios, 29)];
+	       T2D = rio[WS(ios, 21)];
+	       T1w = iio[-WS(ios, 21)];
+	       T1z = rio[WS(ios, 26)];
+	       T2A = iio[-WS(ios, 26)];
+	       T1B = iio[-WS(ios, 5)];
+	       T2z = rio[WS(ios, 5)];
+	       T2F = iio[-WS(ios, 10)];
+	       T1u = rio[WS(ios, 10)];
+	       T2N = rio[WS(ios, 13)];
+	       T1q = iio[-WS(ios, 13)];
+	       T1I = iio[-WS(ios, 1)];
+	       T2O = iio[-WS(ios, 18)];
+	       T1o = rio[WS(ios, 18)];
+	       T1G = rio[WS(ios, 30)];
+	       T2m = iio[-WS(ios, 22)];
+	       T24 = rio[WS(ios, 22)];
+	       T1Z = iio[-WS(ios, 25)];
+	       T2r = rio[WS(ios, 25)];
+	       T1X = rio[WS(ios, 6)];
+	       T2v = iio[-WS(ios, 6)];
+	       T2l = rio[WS(ios, 9)];
+	       T26 = iio[-WS(ios, 9)];
+	       T2i = iio[-WS(ios, 14)];
+	       T1R = rio[WS(ios, 14)];
+	       T1T = iio[-WS(ios, 17)];
+	       T2g = rio[WS(ios, 17)];
+	       {
+		    E T2c, T2d, T3s, T3r, T3j, T3i, T4, T7, T5, T8, T6, T9, T14, T1d, Ts;
+		    E T18, T19, T1c, Te, Td, Tt, Tw, T13, TZ, T10, Tx;
+		    T2c = rio[WS(ios, 1)];
+		    T2d = iio[-WS(ios, 30)];
+		    T3s = iio[-WS(ios, 4)];
+		    T3r = rio[WS(ios, 27)];
+		    T3j = iio[-WS(ios, 28)];
+		    T3i = rio[WS(ios, 3)];
+		    T2 = W[6];
+		    Ti = W[7];
+		    T3 = W[4];
+		    Tc = W[5];
+		    T4 = W[2];
+		    T7 = W[3];
+		    T5 = W[0];
+		    T8 = W[1];
+		    T6 = T4 * T5;
+		    T9 = T7 * T8;
+		    T14 = Ti * T5;
+		    T1d = Tc * T4;
+		    Ts = T3 * T5;
+		    T18 = T3 * T4;
+		    T19 = Tc * T7;
+		    T1c = T3 * T7;
+		    Te = T7 * T5;
+		    Td = T4 * T8;
+		    Tt = Tc * T8;
+		    Tw = T3 * T8;
+		    TF = T2 * T7;
+		    T13 = T2 * T8;
+		    TC = Ti * T7;
+		    TG = Ti * T4;
+		    TZ = T2 * T5;
+		    T10 = Ti * T8;
+		    TB = T2 * T4;
+		    Tx = Tc * T5;
+		    Tu = Ts + Tt;
+		    T1a = T18 - T19;
+		    T15 = T13 + T14;
+		    Ty = Tw - Tx;
+		    T1t = Ts - Tt;
+		    T1Y = T1c - T1d;
+		    T1W = T18 + T19;
+		    T1v = Tw + Tx;
+		    TH = TF - TG;
+		    T1y = TZ + T10;
+		    T11 = TZ - T10;
+		    TD = TB + TC;
+		    T1A = T13 - T14;
+		    T1e = T1c + T1d;
+		    T3t = FMA(T2, T3r, Ti * T3s);
+		    T4g = FNMS(T8, T2c, T5 * T2d);
+		    T4S = FNMS(Ti, T3r, T2 * T3s);
+		    T3k = FMA(T4, T3i, T7 * T3j);
+		    T1n = FMA(T2, T3, Ti * Tc);
+		    T1p = FNMS(Ti, T3, T2 * Tc);
+		    T2e = FMA(T5, T2c, T8 * T2d);
+		    T4M = FNMS(T7, T3i, T4 * T3j);
+		    TM = T6 - T9;
+		    T1K = T3 * TM;
+		    T1O = Tc * TM;
+		    TO = Td + Te;
+		    T1L = Tc * TO;
+		    T1N = T3 * TO;
+		    Ta = T6 + T9;
+		    Tb = T3 * Ta;
+		    T2t = Ti * Ta;
+		    Tk = Tc * Ta;
+		    T2o = T2 * Ta;
+		    Tf = Td - Te;
+		    Tg = Tc * Tf;
+		    T2s = T2 * Tf;
+		    Tj = T3 * Tf;
+		    T2p = Ti * Tf;
+	       }
+	       TQ = FMA(TM, TN, TO * TP);
+	       T3G = FNMS(TO, TN, TM * TP);
+	       T49 = FMA(T1Y, T1X, T1W * T1Z);
+	       T20 = FNMS(T1Y, T1Z, T1W * T1X);
+	       T2n = FMA(T3, T2l, Tc * T2m);
+	       T4y = FNMS(Tc, T2l, T3 * T2m);
+	       {
+		    E T1F, T1H, TA, TJ;
+		    T1F = TB - TC;
+		    T1H = TF + TG;
+		    T1J = FMA(T1F, T1G, T1H * T1I);
+		    T43 = FNMS(T1H, T1G, T1F * T1I);
+		    {
+			 E T2q, T2u, T32, T34;
+			 T2q = T2o - T2p;
+			 T2u = T2s + T2t;
+			 T2w = FMA(T2q, T2r, T2u * T2v);
+			 T4z = FNMS(T2u, T2r, T2q * T2v);
+			 T32 = FMA(T2, T1a, Ti * T1e);
+			 T34 = FNMS(Ti, T1a, T2 * T1e);
+			 T36 = FNMS(T34, T35, T32 * T33);
+			 T4Z = FMA(T34, T33, T32 * T35);
+		    }
+		    TA = FNMS(Ty, Tz, Tu * Tv);
+		    TJ = FNMS(TH, TI, TD * TE);
+		    TK = TA + TJ;
+		    T8b = TA - TJ;
+		    {
+			 E T3Y, T3Z, T3S, T3T;
+			 T3Y = FNMS(T1v, T1u, T1t * T1w);
+			 T3Z = FMA(T1A, T1z, T1y * T1B);
+			 T40 = T3Y - T3Z;
+			 T6l = T3Y + T3Z;
+			 T3S = FMA(Tf, T1k, Ta * T1l);
+			 T3T = FMA(T1p, T1o, T1n * T1q);
+			 T3U = T3S - T3T;
+			 T6k = T3S + T3T;
+		    }
+	       }
+	       {
+		    E T17, T1g, Th, Tl;
+		    T17 = FMA(T11, T12, T15 * T16);
+		    T1g = FMA(T1a, T1b, T1e * T1f);
+		    T1h = T17 + T1g;
+		    T3L = T17 - T1g;
+		    {
+			 E T1x, T1C, T1m, T1r;
+			 T1x = FMA(T1t, T1u, T1v * T1w);
+			 T1C = FNMS(T1A, T1B, T1y * T1z);
+			 T1D = T1x + T1C;
+			 T3V = T1x - T1C;
+			 T1m = FNMS(Tf, T1l, Ta * T1k);
+			 T1r = FNMS(T1p, T1q, T1n * T1o);
+			 T1s = T1m + T1r;
+			 T3X = T1m - T1r;
+		    }
+		    {
+			 E T3C, T3D, T3M, T3N;
+			 T3C = FMA(Ty, Tv, Tu * Tz);
+			 T3D = FMA(TH, TE, TD * TI);
+			 T3E = T3C - T3D;
+			 T7E = T3C + T3D;
+			 T3M = FNMS(T15, T12, T11 * T16);
+			 T3N = FNMS(T1e, T1b, T1a * T1f);
+			 T3O = T3M - T3N;
+			 T6h = T3M + T3N;
+			 {
+			      E T2j, T4h, T2f, T2h;
+			      T2f = FMA(T2, T1t, Ti * T1v);
+			      T2h = FNMS(Ti, T1t, T2 * T1v);
+			      T2j = FNMS(T2h, T2i, T2f * T2g);
+			      T4h = FMA(T2h, T2g, T2f * T2i);
+			      T2k = T2e + T2j;
+			      T6w = T4g + T4h;
+			      T4i = T4g - T4h;
+			      T4x = T2e - T2j;
+			 }
+		    }
+		    {
+			 E T3p, T4N, T3l, T3n;
+			 T3l = FNMS(Ti, Ty, T2 * Tu);
+			 T3n = FMA(T2, Ty, Ti * Tu);
+			 T3p = FMA(T3l, T3m, T3n * T3o);
+			 T4N = FNMS(T3n, T3m, T3l * T3o);
+			 T3q = T3k + T3p;
+			 T6I = T4M + T4N;
+			 T4O = T4M - T4N;
+			 T4P = T3k - T3p;
+		    }
+		    Th = Tb + Tg;
+		    Tl = Tj - Tk;
+		    T3w = FNMS(Tl, T3v, Th * T3u);
+		    T4T = FMA(Tl, T3u, Th * T3v);
+		    T4R = T3t - T3w;
+		    T4U = T4S - T4T;
+		    Tm = FNMS(Ti, Tl, T2 * Th);
+		    To = FMA(T2, Tl, Ti * Th);
+		    {
+			 E TR, TS, TT, TV;
+			 TR = Tb - Tg;
+			 TS = Tj + Tk;
+			 TT = FMA(T2, TR, Ti * TS);
+			 TV = FNMS(Ti, TR, T2 * TS);
+			 TX = FNMS(TV, TW, TT * TU);
+			 T4I = FNMS(TS, T38, TR * T39);
+			 T3a = FMA(TR, T38, TS * T39);
+			 T3H = FMA(TV, TU, TT * TW);
+		    }
+		    {
+			 E T2V, T3b, T2Z, T3d;
+			 {
+			      E T2T, T2U, T2X, T2Y;
+			      T2T = T2 * TM;
+			      T2U = Ti * TO;
+			      T2V = T2T - T2U;
+			      T3b = T2T + T2U;
+			      T2X = T2 * TO;
+			      T2Y = Ti * TM;
+			      T2Z = T2X + T2Y;
+			      T3d = T2X - T2Y;
+			 }
+			 T31 = FMA(T2V, T2W, T2Z * T30);
+			 T4Y = FNMS(T2Z, T2W, T2V * T30);
+			 T3f = FNMS(T3d, T3e, T3b * T3c);
+			 T4J = FMA(T3d, T3c, T3b * T3e);
+		    }
+		    {
+			 E T23, T25, T1Q, T1S;
+			 {
+			      E T2C, T2E, T21, T22;
+			      T2C = FNMS(Ti, T1Y, T2 * T1W);
+			      T2E = FMA(T2, T1Y, Ti * T1W);
+			      T2G = FMA(T2C, T2D, T2E * T2F);
+			      T4s = FNMS(T2E, T2D, T2C * T2F);
+			      T21 = T1K + T1L;
+			      T22 = T1N - T1O;
+			      T23 = FNMS(Ti, T22, T2 * T21);
+			      T4r = FMA(T22, T2z, T21 * T2A);
+			      T25 = FMA(T2, T22, Ti * T21);
+			      T2B = FNMS(T22, T2A, T21 * T2z);
+			 }
+			 T4q = T2B - T2G;
+			 T4t = T4r - T4s;
+			 T27 = FMA(T23, T24, T25 * T26);
+			 T4a = FNMS(T25, T24, T23 * T26);
+			 {
+			      E T2I, T2K, T1M, T1P;
+			      T2I = T2o + T2p;
+			      T2K = T2s - T2t;
+			      T2M = FNMS(T2K, T2L, T2I * T2J);
+			      T4m = FMA(T2K, T2J, T2I * T2L);
+			      T1M = T1K - T1L;
+			      T1P = T1N + T1O;
+			      T1Q = FMA(T2, T1M, Ti * T1P);
+			      T4n = FNMS(T1P, T2N, T1M * T2O);
+			      T1S = FNMS(Ti, T1M, T2 * T1P);
+			      T2P = FMA(T1M, T2N, T1P * T2O);
+			 }
+			 T4l = T2M - T2P;
+			 T4o = T4m - T4n;
+			 T1U = FNMS(T1S, T1T, T1Q * T1R);
+			 T44 = FMA(T1S, T1R, T1Q * T1T);
+		    }
+	       }
+	  }
+	  Tn = rio[WS(ios, 16)];
+	  Tp = iio[-WS(ios, 15)];
+	  T7G = iio[-WS(ios, 31)];
+	  {
+	       E T1i, T7V, T6i, T7D, T42, T5e, T5A, T60, T6o, T6Y, TL, T6f, T3F, T5t, T7I;
+	       E T8q, T7W, T8c, T3Q, T8p, T5w, T89, T4d, T61, T5f, T5D, T2a, T6t, T7O, T7C;
+	       E T7g, T6Z, T4w, T64, T65, T4F, T5i, T5I, T5L, T5j, T2S, T7l, T7y, T6A, T6F;
+	       E T73, T7i, T72, T4X, T67, T68, T56, T5l, T5P, T5S, T5m, T3z, T7q, T7z, T6L;
+	       E T6Q, T76, T7n, T75;
+	       {
+		    E TY, T6g, T3W, T41;
+		    TY = TQ + TX;
+		    T1i = TY + T1h;
+		    T7V = T1h - TY;
+		    T6g = T3G + T3H;
+		    T6i = T6g - T6h;
+		    T7D = T6g + T6h;
+		    T3W = T3U + T3V;
+		    T41 = T3X - T40;
+		    T42 = FNMS(KP923879532, T41, KP382683432 * T3W);
+		    T5e = FMA(KP923879532, T3W, KP382683432 * T41);
+	       }
+	       {
+		    E T5y, T5z, T6m, T6n;
+		    T5y = T3U - T3V;
+		    T5z = T3X + T40;
+		    T5A = FNMS(KP382683432, T5z, KP923879532 * T5y);
+		    T60 = FMA(KP382683432, T5y, KP923879532 * T5z);
+		    T6m = T6k - T6l;
+		    T6n = T1s - T1D;
+		    T6o = T6m - T6n;
+		    T6Y = T6n + T6m;
+	       }
+	       {
+		    E Tr, T3B, Tq, T7H, T8a, T7F;
+		    Tq = FMA(Tm, Tn, To * Tp);
+		    Tr = T1 + Tq;
+		    T3B = T1 - Tq;
+		    TL = Tr + TK;
+		    T6f = Tr - TK;
+		    T3F = T3B - T3E;
+		    T5t = T3B + T3E;
+		    T7F = FNMS(To, Tn, Tm * Tp);
+		    T7H = T7F + T7G;
+		    T8a = T7G - T7F;
+		    T7I = T7E + T7H;
+		    T8q = T8b + T8a;
+		    T7W = T7H - T7E;
+		    T8c = T8a - T8b;
+	       }
+	       {
+		    E T3P, T5v, T3K, T5u, T3I, T3J;
+		    T3P = T3L + T3O;
+		    T5v = T3L - T3O;
+		    T3I = T3G - T3H;
+		    T3J = TQ - TX;
+		    T3K = T3I - T3J;
+		    T5u = T3J + T3I;
+		    T3Q = KP707106781 * (T3K - T3P);
+		    T8p = KP707106781 * (T5v - T5u);
+		    T5w = KP707106781 * (T5u + T5v);
+		    T89 = KP707106781 * (T3K + T3P);
+	       }
+	       {
+		    E T47, T5B, T4c, T5C;
+		    {
+			 E T45, T46, T48, T4b;
+			 T45 = T43 - T44;
+			 T46 = T20 - T27;
+			 T47 = T45 + T46;
+			 T5B = T45 - T46;
+			 T48 = T1J - T1U;
+			 T4b = T49 - T4a;
+			 T4c = T48 - T4b;
+			 T5C = T48 + T4b;
+		    }
+		    T4d = FMA(KP382683432, T47, KP923879532 * T4c);
+		    T61 = FNMS(KP382683432, T5B, KP923879532 * T5C);
+		    T5f = FNMS(KP923879532, T47, KP382683432 * T4c);
+		    T5D = FMA(KP923879532, T5B, KP382683432 * T5C);
+	       }
+	       {
+		    E T1E, T7e, T29, T6p, T6s, T7f;
+		    T1E = T1s + T1D;
+		    T7e = T6k + T6l;
+		    {
+			 E T1V, T28, T6q, T6r;
+			 T1V = T1J + T1U;
+			 T28 = T20 + T27;
+			 T29 = T1V + T28;
+			 T6p = T1V - T28;
+			 T6q = T43 + T44;
+			 T6r = T49 + T4a;
+			 T6s = T6q - T6r;
+			 T7f = T6q + T6r;
+		    }
+		    T2a = T1E + T29;
+		    T6t = T6p + T6s;
+		    T7O = T29 - T1E;
+		    T7C = T7e + T7f;
+		    T7g = T7e - T7f;
+		    T6Z = T6p - T6s;
+	       }
+	       {
+		    E T4k, T5J, T4B, T5G, T4v, T5H, T4E, T5K, T4j, T4A;
+		    T4j = T2n - T2w;
+		    T4k = T4i + T4j;
+		    T5J = T4i - T4j;
+		    T4A = T4y - T4z;
+		    T4B = T4x - T4A;
+		    T5G = T4x + T4A;
+		    {
+			 E T4p, T4u, T4C, T4D;
+			 T4p = T4l - T4o;
+			 T4u = T4q + T4t;
+			 T4v = KP707106781 * (T4p - T4u);
+			 T5H = KP707106781 * (T4u + T4p);
+			 T4C = T4t - T4q;
+			 T4D = T4l + T4o;
+			 T4E = KP707106781 * (T4C - T4D);
+			 T5K = KP707106781 * (T4C + T4D);
+		    }
+		    T4w = T4k - T4v;
+		    T64 = T5G + T5H;
+		    T65 = T5J + T5K;
+		    T4F = T4B - T4E;
+		    T5i = T4k + T4v;
+		    T5I = T5G - T5H;
+		    T5L = T5J - T5K;
+		    T5j = T4B + T4E;
+	       }
+	       {
+		    E T2y, T6B, T6y, T7j, T2R, T6z, T6E, T7k, T2x, T6x;
+		    T2x = T2n + T2w;
+		    T2y = T2k + T2x;
+		    T6B = T2k - T2x;
+		    T6x = T4y + T4z;
+		    T6y = T6w - T6x;
+		    T7j = T6w + T6x;
+		    {
+			 E T2H, T2Q, T6C, T6D;
+			 T2H = T2B + T2G;
+			 T2Q = T2M + T2P;
+			 T2R = T2H + T2Q;
+			 T6z = T2Q - T2H;
+			 T6C = T4r + T4s;
+			 T6D = T4m + T4n;
+			 T6E = T6C - T6D;
+			 T7k = T6C + T6D;
+		    }
+		    T2S = T2y + T2R;
+		    T7l = T7j - T7k;
+		    T7y = T7j + T7k;
+		    T6A = T6y - T6z;
+		    T6F = T6B - T6E;
+		    T73 = T6B + T6E;
+		    T7i = T2y - T2R;
+		    T72 = T6y + T6z;
+	       }
+	       {
+		    E T4L, T5N, T55, T5O, T4W, T5R, T52, T5Q;
+		    {
+			 E T4H, T4K, T53, T54;
+			 T4H = T31 - T36;
+			 T4K = T4I - T4J;
+			 T4L = T4H - T4K;
+			 T5N = T4H + T4K;
+			 T53 = T4R - T4U;
+			 T54 = T4P + T4O;
+			 T55 = KP707106781 * (T53 - T54);
+			 T5O = KP707106781 * (T54 + T53);
+		    }
+		    {
+			 E T4Q, T4V, T50, T51;
+			 T4Q = T4O - T4P;
+			 T4V = T4R + T4U;
+			 T4W = KP707106781 * (T4Q - T4V);
+			 T5R = KP707106781 * (T4Q + T4V);
+			 T50 = T4Y - T4Z;
+			 T51 = T3a - T3f;
+			 T52 = T50 + T51;
+			 T5Q = T50 - T51;
+		    }
+		    T4X = T4L - T4W;
+		    T67 = T5N + T5O;
+		    T68 = T5Q + T5R;
+		    T56 = T52 - T55;
+		    T5l = T4L + T4W;
+		    T5P = T5N - T5O;
+		    T5S = T5Q - T5R;
+		    T5m = T52 + T55;
+	       }
+	       {
+		    E T3y, T6P, T6K, T7p, T3h, T6H, T6O, T7o, T3x, T6J;
+		    T3x = T3t + T3w;
+		    T3y = T3q + T3x;
+		    T6P = T3x - T3q;
+		    T6J = T4S + T4T;
+		    T6K = T6I - T6J;
+		    T7p = T6I + T6J;
+		    {
+			 E T37, T3g, T6M, T6N;
+			 T37 = T31 + T36;
+			 T3g = T3a + T3f;
+			 T3h = T37 + T3g;
+			 T6H = T37 - T3g;
+			 T6M = T4Y + T4Z;
+			 T6N = T4I + T4J;
+			 T6O = T6M - T6N;
+			 T7o = T6M + T6N;
+		    }
+		    T3z = T3h + T3y;
+		    T7q = T7o - T7p;
+		    T7z = T7o + T7p;
+		    T6L = T6H - T6K;
+		    T6Q = T6O - T6P;
+		    T76 = T6O + T6P;
+		    T7n = T3h - T3y;
+		    T75 = T6H + T6K;
+	       }
+	       {
+		    E T3A, T7A, T2b, T7x, T1j;
+		    T3A = T2S + T3z;
+		    T7A = T7y - T7z;
+		    T1j = TL + T1i;
+		    T2b = T1j + T2a;
+		    T7x = T1j - T2a;
+		    iio[-WS(ios, 16)] = T2b - T3A;
+		    rio[WS(ios, 8)] = T7x + T7A;
+		    rio[0] = T2b + T3A;
+		    iio[-WS(ios, 24)] = T7x - T7A;
+	       }
+	       {
+		    E T7B, T7L, T7K, T7M, T7J;
+		    T7B = T7y + T7z;
+		    T7L = T3z - T2S;
+		    T7J = T7D + T7I;
+		    T7K = T7C + T7J;
+		    T7M = T7J - T7C;
+		    rio[WS(ios, 16)] = T7B - T7K;
+		    iio[-WS(ios, 8)] = T7L + T7M;
+		    iio[0] = T7B + T7K;
+		    rio[WS(ios, 24)] = T7L - T7M;
+	       }
+	       {
+		    E T7h, T7t, T7Q, T7S, T7s, T7R, T7w, T7N, T7d, T7P;
+		    T7d = TL - T1i;
+		    T7h = T7d + T7g;
+		    T7t = T7d - T7g;
+		    T7P = T7I - T7D;
+		    T7Q = T7O + T7P;
+		    T7S = T7P - T7O;
+		    {
+			 E T7m, T7r, T7u, T7v;
+			 T7m = T7i + T7l;
+			 T7r = T7n - T7q;
+			 T7s = KP707106781 * (T7m + T7r);
+			 T7R = KP707106781 * (T7r - T7m);
+			 T7u = T7l - T7i;
+			 T7v = T7n + T7q;
+			 T7w = KP707106781 * (T7u - T7v);
+			 T7N = KP707106781 * (T7u + T7v);
+		    }
+		    iio[-WS(ios, 20)] = T7h - T7s;
+		    rio[WS(ios, 20)] = T7N - T7Q;
+		    rio[WS(ios, 4)] = T7h + T7s;
+		    iio[-WS(ios, 4)] = T7N + T7Q;
+		    iio[-WS(ios, 28)] = T7t - T7w;
+		    rio[WS(ios, 28)] = T7R - T7S;
+		    rio[WS(ios, 12)] = T7t + T7w;
+		    iio[-WS(ios, 12)] = T7R + T7S;
+	       }
+	       {
+		    E T71, T79, T7Y, T80, T78, T7Z, T7c, T7T;
+		    {
+			 E T6X, T70, T7U, T7X;
+			 T6X = T6f + T6i;
+			 T70 = KP707106781 * (T6Y + T6Z);
+			 T71 = T6X + T70;
+			 T79 = T6X - T70;
+			 T7U = KP707106781 * (T6o + T6t);
+			 T7X = T7V + T7W;
+			 T7Y = T7U + T7X;
+			 T80 = T7X - T7U;
+		    }
+		    {
+			 E T74, T77, T7a, T7b;
+			 T74 = FMA(KP382683432, T72, KP923879532 * T73);
+			 T77 = FNMS(KP382683432, T76, KP923879532 * T75);
+			 T78 = T74 + T77;
+			 T7Z = T77 - T74;
+			 T7a = FNMS(KP382683432, T73, KP923879532 * T72);
+			 T7b = FMA(KP923879532, T76, KP382683432 * T75);
+			 T7c = T7a - T7b;
+			 T7T = T7a + T7b;
+		    }
+		    iio[-WS(ios, 18)] = T71 - T78;
+		    rio[WS(ios, 18)] = T7T - T7Y;
+		    rio[WS(ios, 2)] = T71 + T78;
+		    iio[-WS(ios, 2)] = T7T + T7Y;
+		    iio[-WS(ios, 26)] = T79 - T7c;
+		    rio[WS(ios, 26)] = T7Z - T80;
+		    rio[WS(ios, 10)] = T79 + T7c;
+		    iio[-WS(ios, 10)] = T7Z + T80;
+	       }
+	       {
+		    E T4f, T59, T8y, T8A, T58, T8z, T5c, T8v;
+		    {
+			 E T3R, T4e, T8w, T8x;
+			 T3R = T3F - T3Q;
+			 T4e = T42 - T4d;
+			 T4f = T3R + T4e;
+			 T59 = T3R - T4e;
+			 T8w = T5f - T5e;
+			 T8x = T8q - T8p;
+			 T8y = T8w + T8x;
+			 T8A = T8x - T8w;
+		    }
+		    {
+			 E T4G, T57, T5a, T5b;
+			 T4G = FMA(KP980785280, T4w, KP195090322 * T4F);
+			 T57 = FNMS(KP980785280, T56, KP195090322 * T4X);
+			 T58 = T4G + T57;
+			 T8z = T57 - T4G;
+			 T5a = FNMS(KP980785280, T4F, KP195090322 * T4w);
+			 T5b = FMA(KP195090322, T56, KP980785280 * T4X);
+			 T5c = T5a - T5b;
+			 T8v = T5a + T5b;
+		    }
+		    iio[-WS(ios, 23)] = T4f - T58;
+		    rio[WS(ios, 23)] = T8v - T8y;
+		    rio[WS(ios, 7)] = T4f + T58;
+		    iio[-WS(ios, 7)] = T8v + T8y;
+		    iio[-WS(ios, 31)] = T59 - T5c;
+		    rio[WS(ios, 31)] = T8z - T8A;
+		    rio[WS(ios, 15)] = T59 + T5c;
+		    iio[-WS(ios, 15)] = T8z + T8A;
+	       }
+	       {
+		    E T5F, T5V, T8k, T8m, T5U, T8l, T5Y, T8h;
+		    {
+			 E T5x, T5E, T8i, T8j;
+			 T5x = T5t - T5w;
+			 T5E = T5A - T5D;
+			 T5F = T5x + T5E;
+			 T5V = T5x - T5E;
+			 T8i = T61 - T60;
+			 T8j = T8c - T89;
+			 T8k = T8i + T8j;
+			 T8m = T8j - T8i;
+		    }
+		    {
+			 E T5M, T5T, T5W, T5X;
+			 T5M = FMA(KP555570233, T5I, KP831469612 * T5L);
+			 T5T = FNMS(KP831469612, T5S, KP555570233 * T5P);
+			 T5U = T5M + T5T;
+			 T8l = T5T - T5M;
+			 T5W = FNMS(KP831469612, T5I, KP555570233 * T5L);
+			 T5X = FMA(KP831469612, T5P, KP555570233 * T5S);
+			 T5Y = T5W - T5X;
+			 T8h = T5W + T5X;
+		    }
+		    iio[-WS(ios, 21)] = T5F - T5U;
+		    rio[WS(ios, 21)] = T8h - T8k;
+		    rio[WS(ios, 5)] = T5F + T5U;
+		    iio[-WS(ios, 5)] = T8h + T8k;
+		    iio[-WS(ios, 29)] = T5V - T5Y;
+		    rio[WS(ios, 29)] = T8l - T8m;
+		    rio[WS(ios, 13)] = T5V + T5Y;
+		    iio[-WS(ios, 13)] = T8l + T8m;
+	       }
+	       {
+		    E T6v, T6T, T84, T86, T6S, T85, T6W, T81;
+		    {
+			 E T6j, T6u, T82, T83;
+			 T6j = T6f - T6i;
+			 T6u = KP707106781 * (T6o - T6t);
+			 T6v = T6j + T6u;
+			 T6T = T6j - T6u;
+			 T82 = KP707106781 * (T6Z - T6Y);
+			 T83 = T7W - T7V;
+			 T84 = T82 + T83;
+			 T86 = T83 - T82;
+		    }
+		    {
+			 E T6G, T6R, T6U, T6V;
+			 T6G = FMA(KP923879532, T6A, KP382683432 * T6F);
+			 T6R = FNMS(KP923879532, T6Q, KP382683432 * T6L);
+			 T6S = T6G + T6R;
+			 T85 = T6R - T6G;
+			 T6U = FNMS(KP923879532, T6F, KP382683432 * T6A);
+			 T6V = FMA(KP382683432, T6Q, KP923879532 * T6L);
+			 T6W = T6U - T6V;
+			 T81 = T6U + T6V;
+		    }
+		    iio[-WS(ios, 22)] = T6v - T6S;
+		    rio[WS(ios, 22)] = T81 - T84;
+		    rio[WS(ios, 6)] = T6v + T6S;
+		    iio[-WS(ios, 6)] = T81 + T84;
+		    iio[-WS(ios, 30)] = T6T - T6W;
+		    rio[WS(ios, 30)] = T85 - T86;
+		    rio[WS(ios, 14)] = T6T + T6W;
+		    iio[-WS(ios, 14)] = T85 + T86;
+	       }
+	       {
+		    E T5h, T5p, T8s, T8u, T5o, T8t, T5s, T8n;
+		    {
+			 E T5d, T5g, T8o, T8r;
+			 T5d = T3F + T3Q;
+			 T5g = T5e + T5f;
+			 T5h = T5d + T5g;
+			 T5p = T5d - T5g;
+			 T8o = T42 + T4d;
+			 T8r = T8p + T8q;
+			 T8s = T8o + T8r;
+			 T8u = T8r - T8o;
+		    }
+		    {
+			 E T5k, T5n, T5q, T5r;
+			 T5k = FMA(KP555570233, T5i, KP831469612 * T5j);
+			 T5n = FNMS(KP555570233, T5m, KP831469612 * T5l);
+			 T5o = T5k + T5n;
+			 T8t = T5n - T5k;
+			 T5q = FNMS(KP555570233, T5j, KP831469612 * T5i);
+			 T5r = FMA(KP831469612, T5m, KP555570233 * T5l);
+			 T5s = T5q - T5r;
+			 T8n = T5q + T5r;
+		    }
+		    iio[-WS(ios, 19)] = T5h - T5o;
+		    rio[WS(ios, 19)] = T8n - T8s;
+		    rio[WS(ios, 3)] = T5h + T5o;
+		    iio[-WS(ios, 3)] = T8n + T8s;
+		    iio[-WS(ios, 27)] = T5p - T5s;
+		    rio[WS(ios, 27)] = T8t - T8u;
+		    rio[WS(ios, 11)] = T5p + T5s;
+		    iio[-WS(ios, 11)] = T8t + T8u;
+	       }
+	       {
+		    E T63, T6b, T8e, T8g, T6a, T8f, T6e, T87;
+		    {
+			 E T5Z, T62, T88, T8d;
+			 T5Z = T5t + T5w;
+			 T62 = T60 + T61;
+			 T63 = T5Z + T62;
+			 T6b = T5Z - T62;
+			 T88 = T5A + T5D;
+			 T8d = T89 + T8c;
+			 T8e = T88 + T8d;
+			 T8g = T8d - T88;
+		    }
+		    {
+			 E T66, T69, T6c, T6d;
+			 T66 = FMA(KP980785280, T64, KP195090322 * T65);
+			 T69 = FNMS(KP195090322, T68, KP980785280 * T67);
+			 T6a = T66 + T69;
+			 T8f = T69 - T66;
+			 T6c = FNMS(KP195090322, T64, KP980785280 * T65);
+			 T6d = FMA(KP195090322, T67, KP980785280 * T68);
+			 T6e = T6c - T6d;
+			 T87 = T6c + T6d;
+		    }
+		    iio[-WS(ios, 17)] = T63 - T6a;
+		    rio[WS(ios, 17)] = T87 - T8e;
+		    rio[WS(ios, 1)] = T63 + T6a;
+		    iio[-WS(ios, 1)] = T87 + T8e;
+		    iio[-WS(ios, 25)] = T6b - T6e;
+		    rio[WS(ios, 25)] = T8f - T8g;
+		    rio[WS(ios, 9)] = T6b + T6e;
+		    iio[-WS(ios, 9)] = T8f + T8g;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_COS, 0, 1},
+     {TW_SIN, 0, 1},
+     {TW_COS, 0, 3},
+     {TW_SIN, 0, 3},
+     {TW_COS, 0, 9},
+     {TW_SIN, 0, 9},
+     {TW_COS, 0, 27},
+     {TW_SIN, 0, 27},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 32, "hf2_32", twinstr, {376, 168, 112, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf2_32) (planner *p) {
+     X(khc2hc_dit_register) (p, hf2_32, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf2_4.c b/src/fftw3/rdft/codelets/r2hc/hf2_4.c
new file mode 100644
index 0000000..a676143
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf2_4.c
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:26 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -twiddle-log3 -n 4 -dit -name hf2_4 -include hf.h */
+
+/*
+ * This function contains 24 FP additions, 16 FP multiplications,
+ * (or, 16 additions, 8 multiplications, 8 fused multiply/add),
+ * 21 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf2_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf2_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf2_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf2_4(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 4) {
+	  E T1, Tp, Ta, Te, To, Tl, Tk, Th;
+	  T1 = rio[0];
+	  Tp = iio[-WS(ios, 3)];
+	  {
+	       E T7, Td, T9, Tc, Tg, Tf, T2, T4, T3, T5, T6, T8;
+	       T7 = rio[WS(ios, 2)];
+	       Td = iio[-WS(ios, 2)];
+	       T9 = iio[-WS(ios, 1)];
+	       Tc = rio[WS(ios, 1)];
+	       Tg = iio[0];
+	       Tf = rio[WS(ios, 3)];
+	       T2 = W[2];
+	       T4 = W[3];
+	       T3 = W[0];
+	       T5 = W[1];
+	       T6 = FMA(T2, T3, T4 * T5);
+	       T8 = FNMS(T4, T3, T2 * T5);
+	       Ta = FNMS(T8, T9, T6 * T7);
+	       Te = FMA(T3, Tc, T5 * Td);
+	       To = FMA(T8, T7, T6 * T9);
+	       Tl = FNMS(T4, Tf, T2 * Tg);
+	       Tk = FNMS(T5, Tc, T3 * Td);
+	       Th = FMA(T2, Tf, T4 * Tg);
+	  }
+	  {
+	       E Tb, Ti, Tn, Tq;
+	       Tb = T1 + Ta;
+	       Ti = Te + Th;
+	       iio[-WS(ios, 2)] = Tb - Ti;
+	       rio[0] = Tb + Ti;
+	       Tn = Tk + Tl;
+	       Tq = To + Tp;
+	       rio[WS(ios, 2)] = Tn - Tq;
+	       iio[0] = Tn + Tq;
+	  }
+	  {
+	       E Tj, Tm, Tr, Ts;
+	       Tj = T1 - Ta;
+	       Tm = Tk - Tl;
+	       iio[-WS(ios, 3)] = Tj - Tm;
+	       rio[WS(ios, 1)] = Tj + Tm;
+	       Tr = Th - Te;
+	       Ts = Tp - To;
+	       rio[WS(ios, 3)] = Tr - Ts;
+	       iio[-WS(ios, 1)] = Tr + Ts;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_COS, 0, 1},
+     {TW_SIN, 0, 1},
+     {TW_COS, 0, 3},
+     {TW_SIN, 0, 3},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 4, "hf2_4", twinstr, {16, 8, 8, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf2_4) (planner *p) {
+     X(khc2hc_dit_register) (p, hf2_4, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf2_64.c b/src/fftw3/rdft/codelets/r2hc/hf2_64.c
new file mode 100644
index 0000000..fc2ec21
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf2_64.c
@@ -0,0 +1,1906 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:57 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -twiddle-log3 -n 64 -dit -name hf2_64 -include hf.h */
+
+/*
+ * This function contains 1154 FP additions, 660 FP multiplications,
+ * (or, 880 additions, 386 multiplications, 274 fused multiply/add),
+ * 382 stack variables, and 256 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf2_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf2_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf2_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf2_64(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP290284677, +0.290284677254462367636192375817395274691476278);
+     DK(KP956940335, +0.956940335732208864935797886980269969482849206);
+     DK(KP881921264, +0.881921264348355029712756863660388349508442621);
+     DK(KP471396736, +0.471396736825997648556387625905254377657460319);
+     DK(KP098017140, +0.098017140329560601994195563888641845861136673);
+     DK(KP995184726, +0.995184726672196886244836953109479921575474869);
+     DK(KP773010453, +0.773010453362736960810906609758469800971041293);
+     DK(KP634393284, +0.634393284163645498215171613225493370675687095);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 10) {
+	  E T1, T1g, T91, T7W, T7m, T2O, T4j, T7P, T4P, T8y, T2w, T8t, T2Z, T8e, T48;
+	  E T1z, T7s, T1I, T7t, T8p, Ten, T1Y, T7D, T2t, T7O, T7L, Te6, T3N, T8E, T7A;
+	  E Te0, T4C, TeA, T8S, T9v, T65, Tfi, T9J, Taq, T6K, Tf6, Ta2, Ta5, T73, Tfc;
+	  E Tad, Tag, T3z, T83, T3u, T82, T81, T84, T15, T9K, T68, T7j, T43, T9w, T4F;
+	  E T8G, T5l, TeL, T9k, T9n, T6o, Tf2, T9Q, T9R, T6z, Tf3, T9T, T9W, To, Ts;
+	  E T4o, T8u, T4U, T92, T5a, TeT, T8V, T8Y, T5G, TeG, T97, T9e, T27, T7X, T2T;
+	  E T7E, T7b, Tai, T6T, Ta3, Tf7, Ta8, T7Q, T2H, T2c, T76, Tah, T7F, T4d, T8z;
+	  E TG, TK, T69, T6b, T3b, T87, T5u, T9l, TeM, T9q, T88, T89, T3o, T86, T5P;
+	  E T9f, TeH, T9a, T34, T8f, T1r, T7n, T3S, T8F, T4G, T4I, Tp, T6c, TH, T6a;
+	  E TL, Ti1, T4H, T4J, Tt;
+	  T1 = rio[0];
+	  {
+	       E T12, T67, T14, T66, T6s, T1b, T1f, T6q, T1m, T6x, T1w, T1q, T6v, T6h, T31;
+	       E T1D, T5I, T1y, T6g, T1S, T6m, T1N, T6W, T6Y, T1M, T6k, T1H, T2Y, T5L, T2W;
+	       E T5N, T2b, T74, T2g, T29, T75, T26, T78, T1W, T22, T7a, T6R, T2u, T6P, T2v;
+	       E T6L, T6M, T2E, T2G, T6I, T5Z, T2n, T63, T6G, T2r, T5H, T33, T5E, T2Q, T5z;
+	       E T5C, T2S, T2M, T5q, T3a, T38, T5s, T2N, T5x, T5n, T3l, T5m, T3n, T5h, T5j;
+	       E T3w, T3y, T58, T4a, T3t, T5d, T3r, T5e, T54, T4c, T4Z, T46, T4T, T4X, T47;
+	       E T4l, T4N, T4i, T4g, T4O, T4n, T4R, T4E, T40, T4D, T42, T4y, T4A, T3J, T3L;
+	       E T3R, T3G, T3E, T3P, T2i, Ta, Ty, Tf, Tw, T2, Tj, T3, Tc, T1E, T1B;
+	       E T1F, T1A, T1R, T3x, T2m, T3K, T61, T1V, T60, T3I, T51, T52, T2V, T56, T5X;
+	       E T3v, T55, T2X, T2q, T5W, T4w, T6E, Ta0, T8Q, Tac, T72, Tb, Tg, Th, T3e;
+	       E T3f, T3h, T1a, T2x, T2B, TU, TV, TY, T1e, T2y, T2A, TC, TD, T1u, Tk;
+	       E Tl, Tm, T39, T3U, T3W, T37, T3T, T3X, TQ, TR, TZ, T3c, T3d, T3i, Tx;
+	       E Tz, T1t, TN, TX, T2f, T5V, Tao, T2h, T3D, T4f, T4h, T3F, T3q, T3s;
+	       T12 = rio[WS(ios, 48)];
+	       T67 = iio[-WS(ios, 48)];
+	       T14 = iio[-WS(ios, 15)];
+	       T66 = rio[WS(ios, 15)];
+	       T6s = iio[-WS(ios, 8)];
+	       T1b = rio[WS(ios, 8)];
+	       T1f = iio[-WS(ios, 55)];
+	       T6q = rio[WS(ios, 55)];
+	       T1m = rio[WS(ios, 40)];
+	       T6x = iio[-WS(ios, 40)];
+	       T1w = rio[WS(ios, 56)];
+	       T1q = iio[-WS(ios, 23)];
+	       T6v = rio[WS(ios, 23)];
+	       T6h = iio[-WS(ios, 56)];
+	       T31 = rio[WS(ios, 50)];
+	       T1D = rio[WS(ios, 24)];
+	       T5I = iio[-WS(ios, 50)];
+	       T1y = iio[-WS(ios, 7)];
+	       T6g = rio[WS(ios, 7)];
+	       T1S = rio[WS(ios, 36)];
+	       T6m = iio[-WS(ios, 24)];
+	       T1N = iio[-WS(ios, 59)];
+	       T6W = rio[WS(ios, 59)];
+	       T6Y = iio[-WS(ios, 4)];
+	       T1M = rio[WS(ios, 4)];
+	       T6k = rio[WS(ios, 39)];
+	       T1H = iio[-WS(ios, 39)];
+	       T2Y = iio[-WS(ios, 45)];
+	       T5L = rio[WS(ios, 45)];
+	       T2W = rio[WS(ios, 18)];
+	       T5N = iio[-WS(ios, 18)];
+	       T2b = iio[-WS(ios, 11)];
+	       T74 = rio[WS(ios, 11)];
+	       T2g = rio[WS(ios, 60)];
+	       T29 = rio[WS(ios, 52)];
+	       T75 = iio[-WS(ios, 52)];
+	       T26 = iio[-WS(ios, 43)];
+	       T78 = rio[WS(ios, 43)];
+	       T1W = iio[-WS(ios, 27)];
+	       T22 = rio[WS(ios, 20)];
+	       T7a = iio[-WS(ios, 20)];
+	       T6R = iio[-WS(ios, 12)];
+	       T2u = rio[WS(ios, 12)];
+	       T6P = rio[WS(ios, 51)];
+	       T2v = iio[-WS(ios, 51)];
+	       T6L = rio[WS(ios, 19)];
+	       T6M = iio[-WS(ios, 44)];
+	       T2E = rio[WS(ios, 44)];
+	       T2G = iio[-WS(ios, 19)];
+	       T6I = iio[-WS(ios, 28)];
+	       T5Z = rio[WS(ios, 31)];
+	       T2n = rio[WS(ios, 28)];
+	       T63 = iio[-WS(ios, 32)];
+	       T6G = rio[WS(ios, 35)];
+	       T2r = iio[-WS(ios, 35)];
+	       T5H = rio[WS(ios, 13)];
+	       T33 = iio[-WS(ios, 13)];
+	       T5E = iio[-WS(ios, 34)];
+	       T2Q = rio[WS(ios, 34)];
+	       T5z = iio[-WS(ios, 2)];
+	       T5C = rio[WS(ios, 29)];
+	       T2S = iio[-WS(ios, 29)];
+	       T2M = rio[WS(ios, 2)];
+	       T5q = rio[WS(ios, 53)];
+	       T3a = iio[-WS(ios, 53)];
+	       T38 = rio[WS(ios, 10)];
+	       T5s = iio[-WS(ios, 10)];
+	       T2N = iio[-WS(ios, 61)];
+	       T5x = rio[WS(ios, 61)];
+	       T5n = iio[-WS(ios, 42)];
+	       T3l = rio[WS(ios, 42)];
+	       T5m = rio[WS(ios, 21)];
+	       T3n = iio[-WS(ios, 21)];
+	       T5h = rio[WS(ios, 37)];
+	       T5j = iio[-WS(ios, 26)];
+	       T3w = rio[WS(ios, 26)];
+	       T3y = iio[-WS(ios, 37)];
+	       T58 = iio[-WS(ios, 38)];
+	       T4a = rio[WS(ios, 38)];
+	       T3t = iio[-WS(ios, 5)];
+	       T5d = rio[WS(ios, 5)];
+	       T3r = rio[WS(ios, 58)];
+	       T5e = iio[-WS(ios, 58)];
+	       T54 = rio[WS(ios, 25)];
+	       T4c = iio[-WS(ios, 25)];
+	       T4Z = iio[-WS(ios, 6)];
+	       T46 = rio[WS(ios, 6)];
+	       T4T = iio[-WS(ios, 22)];
+	       T4X = rio[WS(ios, 57)];
+	       T47 = iio[-WS(ios, 57)];
+	       T4l = rio[WS(ios, 22)];
+	       T4N = rio[WS(ios, 9)];
+	       T4i = iio[-WS(ios, 9)];
+	       T4g = rio[WS(ios, 54)];
+	       T4O = iio[-WS(ios, 54)];
+	       T4n = iio[-WS(ios, 41)];
+	       T4R = rio[WS(ios, 41)];
+	       T4E = iio[-WS(ios, 46)];
+	       T40 = rio[WS(ios, 46)];
+	       T4D = rio[WS(ios, 17)];
+	       T42 = iio[-WS(ios, 17)];
+	       T4y = rio[WS(ios, 33)];
+	       T4A = iio[-WS(ios, 30)];
+	       T3J = rio[WS(ios, 30)];
+	       T3L = iio[-WS(ios, 33)];
+	       T3R = iio[-WS(ios, 49)];
+	       T3G = iio[-WS(ios, 1)];
+	       T3E = rio[WS(ios, 62)];
+	       T3P = rio[WS(ios, 14)];
+	       T2i = iio[-WS(ios, 3)];
+	       {
+		    E T4u, T70, T71, T4v, T5T, T6C, T6D, T5U, T4, T7, T5, T8, TO, TP, T1U;
+		    E T2p, T18, T2k, T2l, T2o, TT, TS, T19, T1c, T1T, T1P, T1Q, T1d;
+		    T4u = rio[WS(ios, 1)];
+		    T70 = rio[WS(ios, 27)];
+		    T71 = iio[-WS(ios, 36)];
+		    T4v = iio[-WS(ios, 62)];
+		    T5T = rio[WS(ios, 63)];
+		    T6C = rio[WS(ios, 3)];
+		    T6D = iio[-WS(ios, 60)];
+		    T5U = iio[0];
+		    {
+			 E T6, Te, T9, Td;
+			 T4 = W[2];
+			 T7 = W[3];
+			 T5 = W[0];
+			 T8 = W[1];
+			 T6 = T4 * T5;
+			 Te = T7 * T5;
+			 T9 = T7 * T8;
+			 Td = T4 * T8;
+			 Ta = T6 - T9;
+			 Ty = Td - Te;
+			 Tf = Td + Te;
+			 Tw = T6 + T9;
+			 T2 = W[6];
+			 Tj = W[7];
+			 T3 = W[4];
+			 Tc = W[5];
+			 TO = T3 * T4;
+			 TP = Tc * T7;
+			 T1U = Tj * T3;
+			 T2p = Tj * T5;
+			 T18 = T3 * T5;
+			 T2k = T2 * T5;
+			 T2l = Tj * T8;
+			 T2o = T2 * T8;
+			 TT = Tc * T4;
+			 TS = T3 * T7;
+			 T19 = Tc * T8;
+			 T1c = T3 * T8;
+			 T1E = T2 * T7;
+			 T1T = T2 * Tc;
+			 T1B = Tj * T7;
+			 T1F = Tj * T4;
+			 T1P = T2 * T3;
+			 T1Q = Tj * Tc;
+			 T1A = T2 * T4;
+			 T1d = Tc * T5;
+		    }
+		    T1R = T1P - T1Q;
+		    T3x = T2o - T2p;
+		    T2m = T2k - T2l;
+		    T3K = T1E + T1F;
+		    T61 = Tj * Ta;
+		    T1V = T1T + T1U;
+		    T60 = T2 * Tf;
+		    T3I = T1A - T1B;
+		    T51 = T2 * Tw;
+		    T52 = Tj * Ty;
+		    T2V = T1P + T1Q;
+		    T56 = Tj * Tw;
+		    T5X = Tj * Tf;
+		    T3v = T2k + T2l;
+		    T55 = T2 * Ty;
+		    T2X = T1T - T1U;
+		    T2q = T2o + T2p;
+		    T5W = T2 * Ta;
+		    T4w = FMA(T5, T4u, T8 * T4v);
+		    T6E = FMA(T4, T6C, T7 * T6D);
+		    Ta0 = FNMS(T7, T6C, T4 * T6D);
+		    T8Q = FNMS(T8, T4u, T5 * T4v);
+		    Tac = FNMS(Tj, T70, T2 * T71);
+		    T72 = FMA(T2, T70, Tj * T71);
+		    Tb = T3 * Ta;
+		    Tg = Tc * Tf;
+		    Th = Tb + Tg;
+		    T3e = TS - TT;
+		    T3f = Tj * T3e;
+		    T3h = T2 * T3e;
+		    T1a = T18 + T19;
+		    T2x = T2 * T1a;
+		    T2B = Tj * T1a;
+		    TU = TS + TT;
+		    TV = Tj * TU;
+		    TY = T2 * TU;
+		    T1e = T1c - T1d;
+		    T2y = Tj * T1e;
+		    T2A = T2 * T1e;
+		    TC = T3 * Ty;
+		    TD = Tc * Tw;
+		    T1u = TC + TD;
+		    Tk = T3 * Tf;
+		    Tl = Tc * Ta;
+		    Tm = Tk - Tl;
+		    T39 = T1c + T1d;
+		    T3U = Tj * T39;
+		    T3W = T2 * T39;
+		    T37 = T18 - T19;
+		    T3T = T2 * T37;
+		    T3X = Tj * T37;
+		    TQ = TO - TP;
+		    TR = T2 * TQ;
+		    TZ = Tj * TQ;
+		    T3c = TO + TP;
+		    T3d = T2 * T3c;
+		    T3i = Tj * T3c;
+		    Tx = T3 * Tw;
+		    Tz = Tc * Ty;
+		    T1t = Tx - Tz;
+		    TN = W[8];
+		    TX = W[9];
+		    T2f = FMA(TN, T4, TX * T7);
+		    T5V = FMA(TN, T5T, TX * T5U);
+		    Tao = FNMS(TX, T5T, TN * T5U);
+		    T2h = FNMS(TX, T4, TN * T7);
+		    T3D = FMA(TN, T5, TX * T8);
+		    T4f = FMA(TN, T3, TX * Tc);
+		    T4h = FNMS(TX, T3, TN * Tc);
+		    T3F = FNMS(TX, T5, TN * T8);
+	       }
+	       T1g = FNMS(T1e, T1f, T1a * T1b);
+	       T91 = FNMS(Tc, T4N, T3 * T4O);
+	       T7W = FMA(Ty, T2M, Tw * T2N);
+	       T7m = FMA(T1e, T1b, T1a * T1f);
+	       T2O = FNMS(Ty, T2N, Tw * T2M);
+	       T4j = FNMS(T4h, T4i, T4f * T4g);
+	       T7P = FNMS(TU, T2u, TQ * T2v);
+	       T4P = FMA(T3, T4N, Tc * T4O);
+	       T8y = FMA(T3e, T46, T3c * T47);
+	       T2w = FMA(TQ, T2u, TU * T2v);
+	       {
+		    E T1v, T1x, T1O, T1X;
+		    T8t = FMA(T4h, T4g, T4f * T4i);
+		    T2Z = FNMS(T2X, T2Y, T2V * T2W);
+		    T8e = FMA(T2X, T2W, T2V * T2Y);
+		    T48 = FNMS(T3e, T47, T3c * T46);
+		    T1v = FMA(TN, T1t, TX * T1u);
+		    T1x = FNMS(TX, T1t, TN * T1u);
+		    T1z = FNMS(T1x, T1y, T1v * T1w);
+		    T7s = FMA(T1x, T1w, T1v * T1y);
+		    {
+			 E T1C, T1G, T8n, T8o;
+			 T1C = T1A + T1B;
+			 T1G = T1E - T1F;
+			 T1I = FNMS(T1G, T1H, T1C * T1D);
+			 T7t = FMA(T1G, T1D, T1C * T1H);
+			 T8n = FMA(T3F, T3E, T3D * T3G);
+			 T8o = FNMS(T3K, T3J, T3I * T3L);
+			 T8p = T8n - T8o;
+			 Ten = T8n + T8o;
+		    }
+		    T1O = FMA(Ta, T1M, Tf * T1N);
+		    T1X = FMA(T1R, T1S, T1V * T1W);
+		    T1Y = T1O + T1X;
+		    T7D = T1O - T1X;
+		    {
+			 E T2j, T2s, T7J, T7K;
+			 T2j = FNMS(T2h, T2i, T2f * T2g);
+			 T2s = FMA(T2m, T2n, T2q * T2r);
+			 T2t = T2j + T2s;
+			 T7O = T2j - T2s;
+			 T7J = FMA(T2h, T2g, T2f * T2i);
+			 T7K = FNMS(T2q, T2n, T2m * T2r);
+			 T7L = T7J - T7K;
+			 Te6 = T7J + T7K;
+		    }
+	       }
+	       {
+		    E T3H, T3M, T7y, T7z;
+		    T3H = FNMS(T3F, T3G, T3D * T3E);
+		    T3M = FMA(T3I, T3J, T3K * T3L);
+		    T3N = T3H + T3M;
+		    T8E = T3H - T3M;
+		    T7y = FNMS(Tf, T1M, Ta * T1N);
+		    T7z = FNMS(T1V, T1S, T1R * T1W);
+		    T7A = T7y - T7z;
+		    Te0 = T7y + T7z;
+	       }
+	       {
+		    E T4B, T8R, T4x, T4z;
+		    T4x = T3d + T3f;
+		    T4z = T3h - T3i;
+		    T4B = FNMS(T4z, T4A, T4x * T4y);
+		    T8R = FMA(T4z, T4y, T4x * T4A);
+		    T4C = T4w + T4B;
+		    TeA = T8Q + T8R;
+		    T8S = T8Q - T8R;
+		    T9v = T4w - T4B;
+	       }
+	       {
+		    E T64, Tap, T5Y, T62;
+		    T5Y = T5W - T5X;
+		    T62 = T60 + T61;
+		    T64 = FMA(T5Y, T5Z, T62 * T63);
+		    Tap = FNMS(T62, T5Z, T5Y * T63);
+		    T65 = T5V + T64;
+		    Tfi = Tao + Tap;
+		    T9J = T5V - T64;
+		    Taq = Tao - Tap;
+	       }
+	       {
+		    E T6J, Ta1, T6F, T6H;
+		    T6F = T2x + T2y;
+		    T6H = T2A - T2B;
+		    T6J = FNMS(T6H, T6I, T6F * T6G);
+		    Ta1 = FMA(T6H, T6G, T6F * T6I);
+		    T6K = T6E + T6J;
+		    Tf6 = Ta0 + Ta1;
+		    Ta2 = Ta0 - Ta1;
+		    Ta5 = T6E - T6J;
+	       }
+	       {
+		    E T6Z, Tab, T6V, T6X;
+		    T6V = FMA(TN, Ta, TX * Tf);
+		    T6X = FNMS(TX, Ta, TN * Tf);
+		    T6Z = FNMS(T6X, T6Y, T6V * T6W);
+		    Tab = FMA(T6X, T6W, T6V * T6Y);
+		    T73 = T6Z + T72;
+		    Tfc = Tab + Tac;
+		    Tad = Tab - Tac;
+		    Tag = T6Z - T72;
+	       }
+	       T3z = FNMS(T3x, T3y, T3v * T3w);
+	       T83 = FMA(T3x, T3w, T3v * T3y);
+	       T3q = FNMS(TX, Tm, TN * Th);
+	       T3s = FMA(TN, Tm, TX * Th);
+	       T3u = FMA(T3q, T3r, T3s * T3t);
+	       T82 = FNMS(T3s, T3r, T3q * T3t);
+	       T81 = T3u - T3z;
+	       T84 = T82 - T83;
+	       {
+		    E TW, T10, T11, T13;
+		    TW = TR + TV;
+		    T10 = TY - TZ;
+		    T11 = FNMS(TX, T10, TN * TW);
+		    T13 = FMA(TN, T10, TX * TW);
+		    T15 = FMA(T11, T12, T13 * T14);
+		    T9K = FMA(T10, T66, TW * T67);
+		    T68 = FNMS(T10, T67, TW * T66);
+		    T7j = FNMS(T13, T12, T11 * T14);
+	       }
+	       {
+		    E T3V, T3Y, T3Z, T41;
+		    T3V = T3T + T3U;
+		    T3Y = T3W - T3X;
+		    T3Z = FNMS(TX, T3Y, TN * T3V);
+		    T41 = FMA(TN, T3Y, TX * T3V);
+		    T43 = FMA(T3Z, T40, T41 * T42);
+		    T9w = FMA(T3Y, T4D, T3V * T4E);
+		    T4F = FNMS(T3Y, T4E, T3V * T4D);
+		    T8G = FNMS(T41, T40, T3Z * T42);
+	       }
+	       {
+		    E T5f, T9i, T5k, T9j, T5g, T5i;
+		    T5f = FNMS(Tm, T5e, Th * T5d);
+		    T9i = FMA(Tm, T5d, Th * T5e);
+		    T5g = T3T - T3U;
+		    T5i = T3W + T3X;
+		    T5k = FMA(T5g, T5h, T5i * T5j);
+		    T9j = FNMS(T5i, T5h, T5g * T5j);
+		    T5l = T5f + T5k;
+		    TeL = T9i + T9j;
+		    T9k = T9i - T9j;
+		    T9n = T5f - T5k;
+	       }
+	       {
+		    E T6i, T9O, T6n, T9P, T6j, T6l;
+		    T6i = FMA(T1t, T6g, T1u * T6h);
+		    T9O = FNMS(T1u, T6g, T1t * T6h);
+		    T6j = TR - TV;
+		    T6l = TY + TZ;
+		    T6n = FMA(T6j, T6k, T6l * T6m);
+		    T9P = FNMS(T6l, T6k, T6j * T6m);
+		    T6o = T6i + T6n;
+		    Tf2 = T9O + T9P;
+		    T9Q = T9O - T9P;
+		    T9R = T6i - T6n;
+	       }
+	       {
+		    E T6t, T9U, T6y, T9V;
+		    {
+			 E T6p, T6r, T6u, T6w;
+			 T6p = FNMS(TX, T1e, TN * T1a);
+			 T6r = FMA(TN, T1e, TX * T1a);
+			 T6t = FMA(T6p, T6q, T6r * T6s);
+			 T9U = FNMS(T6r, T6q, T6p * T6s);
+			 T6u = T5W + T5X;
+			 T6w = T60 - T61;
+			 T6y = FNMS(T6w, T6x, T6u * T6v);
+			 T9V = FMA(T6w, T6v, T6u * T6x);
+		    }
+		    T6z = T6t + T6y;
+		    Tf3 = T9U + T9V;
+		    T9T = T6t - T6y;
+		    T9W = T9U - T9V;
+	       }
+	       {
+		    E Ti, Tn, T4k, Tq, Tr, T4m, T4Q, T4S;
+		    Ti = T2 * Th;
+		    Tn = Tj * Tm;
+		    T4k = Ti - Tn;
+		    Tq = T2 * Tm;
+		    Tr = Tj * Th;
+		    T4m = Tq + Tr;
+		    To = Ti + Tn;
+		    Ts = Tq - Tr;
+		    T4o = FMA(T4k, T4l, T4m * T4n);
+		    T8u = FNMS(T4m, T4l, T4k * T4n);
+		    T4Q = FMA(TN, T4k, TX * T4m);
+		    T4S = FNMS(TX, T4k, TN * T4m);
+		    T4U = FNMS(T4S, T4T, T4Q * T4R);
+		    T92 = FMA(T4S, T4R, T4Q * T4T);
+	       }
+	       {
+		    E T50, T8W, T59, T8X;
+		    {
+			 E T4W, T4Y, T53, T57;
+			 T4W = FNMS(TX, T3e, TN * T3c);
+			 T4Y = FMA(TN, T3e, TX * T3c);
+			 T50 = FMA(T4W, T4X, T4Y * T4Z);
+			 T8W = FNMS(T4Y, T4X, T4W * T4Z);
+			 T53 = T51 - T52;
+			 T57 = T55 + T56;
+			 T59 = FMA(T53, T54, T57 * T58);
+			 T8X = FNMS(T57, T54, T53 * T58);
+		    }
+		    T5a = T50 + T59;
+		    TeT = T8W + T8X;
+		    T8V = T50 - T59;
+		    T8Y = T8W - T8X;
+	       }
+	       {
+		    E T5A, T9c, T5F, T9d;
+		    {
+			 E T5w, T5y, T5B, T5D;
+			 T5w = FNMS(TX, Ty, TN * Tw);
+			 T5y = FMA(TN, Ty, TX * Tw);
+			 T5A = FMA(T5w, T5x, T5y * T5z);
+			 T9c = FNMS(T5y, T5x, T5w * T5z);
+			 T5B = T51 + T52;
+			 T5D = T55 - T56;
+			 T5F = FNMS(T5D, T5E, T5B * T5C);
+			 T9d = FMA(T5D, T5C, T5B * T5E);
+		    }
+		    T5G = T5A + T5F;
+		    TeG = T9c + T9d;
+		    T97 = T5A - T5F;
+		    T9e = T9c - T9d;
+	       }
+	       {
+		    E T21, T2P, T25, T2R, T77, T79;
+		    {
+			 E T1Z, T20, T23, T24;
+			 T1Z = T2 * T1t;
+			 T20 = Tj * T1u;
+			 T21 = T1Z + T20;
+			 T2P = T1Z - T20;
+			 T23 = T2 * T1u;
+			 T24 = Tj * T1t;
+			 T25 = T23 - T24;
+			 T2R = T23 + T24;
+		    }
+		    T27 = FNMS(T25, T26, T21 * T22);
+		    T7X = FNMS(T2R, T2Q, T2P * T2S);
+		    T2T = FMA(T2P, T2Q, T2R * T2S);
+		    T7E = FMA(T25, T22, T21 * T26);
+		    T77 = FNMS(TX, T25, TN * T21);
+		    T79 = FMA(TN, T25, TX * T21);
+		    T7b = FMA(T77, T78, T79 * T7a);
+		    Tai = FNMS(T79, T78, T77 * T7a);
+	       }
+	       {
+		    E T6S, Ta7, T2D, Ta6, T2F, T6N;
+		    {
+			 E T6O, T6Q, T2z, T2C;
+			 T6O = FMA(TN, TQ, TX * TU);
+			 T6Q = FNMS(TX, TQ, TN * TU);
+			 T6S = FNMS(T6Q, T6R, T6O * T6P);
+			 Ta7 = FMA(T6Q, T6P, T6O * T6R);
+			 T2z = T2x - T2y;
+			 T2C = T2A + T2B;
+			 T2D = FMA(TN, T2z, TX * T2C);
+			 Ta6 = FNMS(T2C, T6L, T2z * T6M);
+			 T2F = FNMS(TX, T2z, TN * T2C);
+			 T6N = FMA(T2z, T6L, T2C * T6M);
+		    }
+		    T6T = T6N + T6S;
+		    Ta3 = T6N - T6S;
+		    Tf7 = Ta6 + Ta7;
+		    Ta8 = Ta6 - Ta7;
+		    T7Q = FMA(T2F, T2E, T2D * T2G);
+		    T2H = FNMS(T2F, T2G, T2D * T2E);
+	       }
+	       {
+		    E TA, TE, TB, TF, TJ, TI, T2a, T28, T49, T4b;
+		    TA = Tx + Tz;
+		    TE = TC - TD;
+		    TB = T2 * TA;
+		    TF = Tj * TE;
+		    TJ = Tj * TA;
+		    TI = T2 * TE;
+		    T2a = FMA(TN, TE, TX * TA);
+		    T28 = FNMS(TX, TE, TN * TA);
+		    T2c = FMA(T28, T29, T2a * T2b);
+		    T76 = FNMS(TE, T75, TA * T74);
+		    Tah = FMA(TE, T74, TA * T75);
+		    T7F = FNMS(T2a, T29, T28 * T2b);
+		    T49 = TB + TF;
+		    T4b = TI - TJ;
+		    T4d = FNMS(T4b, T4c, T49 * T4a);
+		    T8z = FMA(T4b, T4a, T49 * T4c);
+		    TG = TB - TF;
+		    TK = TI + TJ;
+		    T69 = FMA(TN, TG, TX * TK);
+		    T6b = FNMS(TX, TG, TN * TK);
+	       }
+	       {
+		    E T5t, T9p, T3k, T9o, T3m, T5o;
+		    T3b = FMA(T37, T38, T39 * T3a);
+		    T87 = FNMS(T39, T38, T37 * T3a);
+		    {
+			 E T5p, T5r, T3g, T3j;
+			 T5p = FMA(TN, T37, TX * T39);
+			 T5r = FNMS(TX, T37, TN * T39);
+			 T5t = FNMS(T5r, T5s, T5p * T5q);
+			 T9p = FMA(T5r, T5q, T5p * T5s);
+			 T3g = T3d - T3f;
+			 T3j = T3h + T3i;
+			 T3k = FMA(TN, T3g, TX * T3j);
+			 T9o = FNMS(T3j, T5m, T3g * T5n);
+			 T3m = FNMS(TX, T3g, TN * T3j);
+			 T5o = FMA(T3g, T5m, T3j * T5n);
+		    }
+		    T5u = T5o + T5t;
+		    T9l = T5o - T5t;
+		    TeM = T9o + T9p;
+		    T9q = T9o - T9p;
+		    T88 = FMA(T3m, T3l, T3k * T3n);
+		    T89 = T87 - T88;
+		    T3o = FNMS(T3m, T3n, T3k * T3l);
+		    T86 = T3b - T3o;
+	       }
+	       {
+		    E T5O, T99, T1i, T1n, T1o, T1k, T30, T5J, T98, T32;
+		    {
+			 E T5K, T5M, T1h, T1j;
+			 T5K = FNMS(TX, T2X, TN * T2V);
+			 T5M = FMA(TN, T2X, TX * T2V);
+			 T5O = FMA(T5K, T5L, T5M * T5N);
+			 T99 = FNMS(T5M, T5L, T5K * T5N);
+			 T1h = Tb - Tg;
+			 T1j = Tk + Tl;
+			 T1i = T2 * T1h;
+			 T1n = T2 * T1j;
+			 T1o = Tj * T1h;
+			 T1k = Tj * T1j;
+			 T30 = FMA(TN, T1h, TX * T1j);
+			 T5J = FMA(T1h, T5H, T1j * T5I);
+			 T98 = FNMS(T1j, T5H, T1h * T5I);
+			 T32 = FNMS(TX, T1h, TN * T1j);
+		    }
+		    T5P = T5J + T5O;
+		    T9f = T5J - T5O;
+		    TeH = T98 + T99;
+		    T9a = T98 - T99;
+		    T34 = FNMS(T32, T33, T30 * T31);
+		    T8f = FMA(T32, T31, T30 * T33);
+		    {
+			 E T1l, T1p, T3O, T3Q;
+			 T1l = T1i - T1k;
+			 T1p = T1n + T1o;
+			 T1r = FMA(T1l, T1m, T1p * T1q);
+			 T7n = FNMS(T1p, T1m, T1l * T1q);
+			 T3O = T1i + T1k;
+			 T3Q = T1n - T1o;
+			 T3S = FNMS(T3Q, T3R, T3O * T3P);
+			 T8F = FMA(T3Q, T3P, T3O * T3R);
+			 T4G = FNMS(TX, T3Q, TN * T3O);
+			 T4I = FMA(TN, T3Q, TX * T3O);
+		    }
+	       }
+	  }
+	  Tp = rio[WS(ios, 32)];
+	  T6c = iio[-WS(ios, 16)];
+	  TH = rio[WS(ios, 16)];
+	  T6a = rio[WS(ios, 47)];
+	  TL = iio[-WS(ios, 47)];
+	  Ti1 = iio[-WS(ios, 63)];
+	  T4H = rio[WS(ios, 49)];
+	  T4J = iio[-WS(ios, 14)];
+	  Tt = iio[-WS(ios, 31)];
+	  {
+	       E T5R, TgT, TgY, ThE, T9t, Tbe, T9G, Tbb, Tcl, Tdq, Tcs, Tdn, TeP, Tg4, TeY;
+	       E Tg1, T7e, Th4, ThJ, Th9, Tfp, Tg8, Tfg, Tgb, T2K, TgC, Tih, ThX, TfQ, TiL;
+	       E Tea, Tiv, Tam, Tbl, TcL, Tdu, Taz, Tbi, TcE, Tdx, T7U, Tjv, Tdc, Tjh, Tb0;
+	       E TjL, TbU, TiZ, T8D, Tb5, Tc8, Tdi, T8M, Tb6, Tc5, Tdh, T4r, Thz, Tex, Tfz;
+	       E TfX, Tgl, TgN, Thj, T8m, TaI, Tdg, TdG, Tb4, Tbu, Tc2, TcU, T3C, Thy, Tem;
+	       E Tfy, TfU, Tgk, TgI, Thi, T6B, Th1, Tfm, Tga, Th8, ThI, T9Z, Tbh, Taw, Tbk;
+	       E TcI, Tdw, Tf5, Tg7, Tcx, Tdt, T5c, TgV, TeV, Tg0, TgS, ThD, TeE, Tg3, T96;
+	       E Tbd, Tce, Tdp, Tcp, Tdm, T9D, Tba, T1L, Tgz, Ti4, Tii, Tiy, TiM, TdZ, TfN;
+	       E T7x, TaX, Tj4, Tji, Tjy, TjM, TbN, Td9;
+	       {
+		    E T5v, T5Q, TgW, TgX;
+		    T5v = T5l + T5u;
+		    T5Q = T5G + T5P;
+		    T5R = T5v + T5Q;
+		    TgT = T5Q - T5v;
+		    TgW = TeL + TeM;
+		    TgX = TeG + TeH;
+		    TgY = TgW - TgX;
+		    ThE = TgW + TgX;
+	       }
+	       {
+		    E T9h, T9F, T9s, T9E;
+		    {
+			 E T9b, T9g, T9m, T9r;
+			 T9b = T97 - T9a;
+			 T9g = T9e + T9f;
+			 T9h = FNMS(KP923879532, T9g, KP382683432 * T9b);
+			 T9F = FMA(KP382683432, T9g, KP923879532 * T9b);
+			 T9m = T9k + T9l;
+			 T9r = T9n - T9q;
+			 T9s = FMA(KP923879532, T9m, KP382683432 * T9r);
+			 T9E = FNMS(KP923879532, T9r, KP382683432 * T9m);
+		    }
+		    T9t = T9h - T9s;
+		    Tbe = T9E + T9F;
+		    T9G = T9E - T9F;
+		    Tbb = T9s + T9h;
+	       }
+	       {
+		    E Tch, Tcr, Tck, Tcq;
+		    {
+			 E Tcf, Tcg, Tci, Tcj;
+			 Tcf = T97 + T9a;
+			 Tcg = T9e - T9f;
+			 Tch = FNMS(KP382683432, Tcg, KP923879532 * Tcf);
+			 Tcr = FMA(KP923879532, Tcg, KP382683432 * Tcf);
+			 Tci = T9k - T9l;
+			 Tcj = T9n + T9q;
+			 Tck = FMA(KP382683432, Tci, KP923879532 * Tcj);
+			 Tcq = FNMS(KP382683432, Tcj, KP923879532 * Tci);
+		    }
+		    Tcl = Tch - Tck;
+		    Tdq = Tcq + Tcr;
+		    Tcs = Tcq - Tcr;
+		    Tdn = Tck + Tch;
+	       }
+	       {
+		    E TeJ, TeX, TeO, TeW;
+		    {
+			 E TeF, TeI, TeK, TeN;
+			 TeF = T5G - T5P;
+			 TeI = TeG - TeH;
+			 TeJ = TeF - TeI;
+			 TeX = TeF + TeI;
+			 TeK = T5l - T5u;
+			 TeN = TeL - TeM;
+			 TeO = TeK + TeN;
+			 TeW = TeN - TeK;
+		    }
+		    TeP = KP707106781 * (TeJ - TeO);
+		    Tg4 = KP707106781 * (TeW + TeX);
+		    TeY = KP707106781 * (TeW - TeX);
+		    Tg1 = KP707106781 * (TeO + TeJ);
+	       }
+	       {
+		    E T6U, Th2, T7d, Tfb, Tfe, Th3, Tfa, Tfo, Tfn, Tff;
+		    T6U = T6K + T6T;
+		    Th2 = Tf6 + Tf7;
+		    {
+			 E T7c, Tfd, Tf8, Tf9;
+			 T7c = T76 + T7b;
+			 T7d = T73 + T7c;
+			 Tfb = T73 - T7c;
+			 Tfd = Tah + Tai;
+			 Tfe = Tfc - Tfd;
+			 Th3 = Tfc + Tfd;
+			 Tf8 = Tf6 - Tf7;
+			 Tf9 = T6K - T6T;
+			 Tfa = Tf8 - Tf9;
+			 Tfo = Tf9 + Tf8;
+		    }
+		    T7e = T6U + T7d;
+		    Th4 = Th2 - Th3;
+		    ThJ = Th2 + Th3;
+		    Th9 = T7d - T6U;
+		    Tfn = Tfb - Tfe;
+		    Tfp = KP707106781 * (Tfn - Tfo);
+		    Tg8 = KP707106781 * (Tfo + Tfn);
+		    Tff = Tfb + Tfe;
+		    Tfg = KP707106781 * (Tfa - Tff);
+		    Tgb = KP707106781 * (Tfa + Tff);
+	       }
+	       {
+		    E T2e, Te3, Te8, TgB, T2J, Te5, Te2, TgA;
+		    {
+			 E T2d, Te7, T2I, Te1;
+			 T2d = T27 + T2c;
+			 T2e = T1Y + T2d;
+			 Te3 = T1Y - T2d;
+			 Te7 = T7P + T7Q;
+			 Te8 = Te6 - Te7;
+			 TgB = Te6 + Te7;
+			 T2I = T2w + T2H;
+			 T2J = T2t + T2I;
+			 Te5 = T2t - T2I;
+			 Te1 = T7E + T7F;
+			 Te2 = Te0 - Te1;
+			 TgA = Te0 + Te1;
+		    }
+		    T2K = T2e + T2J;
+		    TgC = TgA - TgB;
+		    Tih = T2J - T2e;
+		    ThX = TgA + TgB;
+		    {
+			 E TfO, TfP, Te4, Te9;
+			 TfO = Te3 + Te2;
+			 TfP = Te5 - Te8;
+			 TfQ = KP707106781 * (TfO + TfP);
+			 TiL = KP707106781 * (TfP - TfO);
+			 Te4 = Te2 - Te3;
+			 Te9 = Te5 + Te8;
+			 Tea = KP707106781 * (Te4 - Te9);
+			 Tiv = KP707106781 * (Te4 + Te9);
+		    }
+	       }
+	       {
+		    E Taf, TcB, Tak, TcC, Taa, Tay, TcA, TcK, Tae, Taj;
+		    Tae = T76 - T7b;
+		    Taf = Tad + Tae;
+		    TcB = Tad - Tae;
+		    Taj = Tah - Tai;
+		    Tak = Tag - Taj;
+		    TcC = Tag + Taj;
+		    {
+			 E Ta4, Ta9, Tcy, Tcz;
+			 Ta4 = Ta2 + Ta3;
+			 Ta9 = Ta5 - Ta8;
+			 Taa = FNMS(KP923879532, Ta9, KP382683432 * Ta4);
+			 Tay = FMA(KP923879532, Ta4, KP382683432 * Ta9);
+			 Tcy = Ta2 - Ta3;
+			 Tcz = Ta5 + Ta8;
+			 TcA = FNMS(KP382683432, Tcz, KP923879532 * Tcy);
+			 TcK = FMA(KP382683432, Tcy, KP923879532 * Tcz);
+		    }
+		    {
+			 E Tal, TcJ, Tax, TcD;
+			 Tal = FMA(KP382683432, Taf, KP923879532 * Tak);
+			 Tam = Taa - Tal;
+			 Tbl = Taa + Tal;
+			 TcJ = FNMS(KP382683432, TcB, KP923879532 * TcC);
+			 TcL = TcJ - TcK;
+			 Tdu = TcK + TcJ;
+			 Tax = FNMS(KP923879532, Taf, KP382683432 * Tak);
+			 Taz = Tax - Tay;
+			 Tbi = Tay + Tax;
+			 TcD = FMA(KP923879532, TcB, KP382683432 * TcC);
+			 TcE = TcA - TcD;
+			 Tdx = TcA + TcD;
+		    }
+	       }
+	       {
+		    E T7C, TbO, T7S, TbS, T7H, TbP, T7N, TbR;
+		    {
+			 E T7B, T7R, T7G, T7M;
+			 T7B = T27 - T2c;
+			 T7C = T7A + T7B;
+			 TbO = T7A - T7B;
+			 T7R = T7P - T7Q;
+			 T7S = T7O - T7R;
+			 TbS = T7O + T7R;
+			 T7G = T7E - T7F;
+			 T7H = T7D - T7G;
+			 TbP = T7D + T7G;
+			 T7M = T2w - T2H;
+			 T7N = T7L + T7M;
+			 TbR = T7L - T7M;
+		    }
+		    {
+			 E T7I, T7T, Tda, Tdb;
+			 T7I = FNMS(KP923879532, T7H, KP382683432 * T7C);
+			 T7T = FMA(KP382683432, T7N, KP923879532 * T7S);
+			 T7U = T7I - T7T;
+			 Tjv = T7I + T7T;
+			 Tda = FMA(KP382683432, TbO, KP923879532 * TbP);
+			 Tdb = FNMS(KP382683432, TbR, KP923879532 * TbS);
+			 Tdc = Tda + Tdb;
+			 Tjh = Tdb - Tda;
+		    }
+		    {
+			 E TaY, TaZ, TbQ, TbT;
+			 TaY = FMA(KP923879532, T7C, KP382683432 * T7H);
+			 TaZ = FNMS(KP923879532, T7N, KP382683432 * T7S);
+			 Tb0 = TaY + TaZ;
+			 TjL = TaZ - TaY;
+			 TbQ = FNMS(KP382683432, TbP, KP923879532 * TbO);
+			 TbT = FMA(KP923879532, TbR, KP382683432 * TbS);
+			 TbU = TbQ - TbT;
+			 TiZ = TbQ + TbT;
+		    }
+	       }
+	       {
+		    E T8r, Tc6, T8I, Tc3, T8w, T8K, T8B, T8J, T8q, T8H;
+		    T8q = T3S - T43;
+		    T8r = T8p + T8q;
+		    Tc6 = T8p - T8q;
+		    T8H = T8F - T8G;
+		    T8I = T8E - T8H;
+		    Tc3 = T8E + T8H;
+		    {
+			 E T8s, T8v, T8x, T8A;
+			 T8s = T4j - T4o;
+			 T8v = T8t - T8u;
+			 T8w = T8s - T8v;
+			 T8K = T8s + T8v;
+			 T8x = T48 - T4d;
+			 T8A = T8y - T8z;
+			 T8B = T8x + T8A;
+			 T8J = T8A - T8x;
+		    }
+		    {
+			 E T8C, Tc7, T8L, Tc4;
+			 T8C = KP707106781 * (T8w - T8B);
+			 T8D = T8r - T8C;
+			 Tb5 = T8r + T8C;
+			 Tc7 = KP707106781 * (T8J + T8K);
+			 Tc8 = Tc6 - Tc7;
+			 Tdi = Tc6 + Tc7;
+			 T8L = KP707106781 * (T8J - T8K);
+			 T8M = T8I - T8L;
+			 Tb6 = T8I + T8L;
+			 Tc4 = KP707106781 * (T8B + T8w);
+			 Tc5 = Tc3 - Tc4;
+			 Tdh = Tc3 + Tc4;
+		    }
+	       }
+	       {
+		    E T45, Tes, Tep, TgK, T4q, Teq, Tev, TgL, T44, Teo, Ter, Tew;
+		    T44 = T3S + T43;
+		    T45 = T3N + T44;
+		    Tes = T3N - T44;
+		    Teo = T8F + T8G;
+		    Tep = Ten - Teo;
+		    TgK = Ten + Teo;
+		    {
+			 E T4e, T4p, Tet, Teu;
+			 T4e = T48 + T4d;
+			 T4p = T4j + T4o;
+			 T4q = T4e + T4p;
+			 Teq = T4p - T4e;
+			 Tet = T8y + T8z;
+			 Teu = T8t + T8u;
+			 Tev = Tet - Teu;
+			 TgL = Tet + Teu;
+		    }
+		    T4r = T45 + T4q;
+		    Thz = TgK + TgL;
+		    Ter = Tep - Teq;
+		    Tew = Tes - Tev;
+		    Tex = FMA(KP382683432, Ter, KP923879532 * Tew);
+		    Tfz = FNMS(KP923879532, Ter, KP382683432 * Tew);
+		    {
+			 E TfV, TfW, TgJ, TgM;
+			 TfV = Tep + Teq;
+			 TfW = Tes + Tev;
+			 TfX = FMA(KP923879532, TfV, KP382683432 * TfW);
+			 Tgl = FNMS(KP382683432, TfV, KP923879532 * TfW);
+			 TgJ = T45 - T4q;
+			 TgM = TgK - TgL;
+			 TgN = TgJ + TgM;
+			 Thj = TgJ - TgM;
+		    }
+	       }
+	       {
+		    E T80, TbW, T8k, TbX, T8b, Tc0, T8h, TbZ;
+		    {
+			 E T7Y, T7Z, T8i, T8j;
+			 T7Y = T7W - T7X;
+			 T7Z = T2Z - T34;
+			 T80 = T7Y + T7Z;
+			 TbW = T7Y - T7Z;
+			 T8i = T89 - T86;
+			 T8j = T81 + T84;
+			 T8k = KP707106781 * (T8i - T8j);
+			 TbX = KP707106781 * (T8i + T8j);
+		    }
+		    {
+			 E T85, T8a, T8d, T8g;
+			 T85 = T81 - T84;
+			 T8a = T86 + T89;
+			 T8b = KP707106781 * (T85 - T8a);
+			 Tc0 = KP707106781 * (T8a + T85);
+			 T8d = T2O - T2T;
+			 T8g = T8e - T8f;
+			 T8h = T8d - T8g;
+			 TbZ = T8d + T8g;
+		    }
+		    {
+			 E T8c, T8l, Tde, Tdf;
+			 T8c = T80 - T8b;
+			 T8l = T8h - T8k;
+			 T8m = FNMS(KP980785280, T8l, KP195090322 * T8c);
+			 TaI = FMA(KP980785280, T8c, KP195090322 * T8l);
+			 Tde = TbW + TbX;
+			 Tdf = TbZ + Tc0;
+			 Tdg = FNMS(KP195090322, Tdf, KP980785280 * Tde);
+			 TdG = FMA(KP980785280, Tdf, KP195090322 * Tde);
+		    }
+		    {
+			 E Tb2, Tb3, TbY, Tc1;
+			 Tb2 = T80 + T8b;
+			 Tb3 = T8h + T8k;
+			 Tb4 = FNMS(KP555570233, Tb3, KP831469612 * Tb2);
+			 Tbu = FMA(KP555570233, Tb2, KP831469612 * Tb3);
+			 TbY = TbW - TbX;
+			 Tc1 = TbZ - Tc0;
+			 Tc2 = FNMS(KP831469612, Tc1, KP555570233 * TbY);
+			 TcU = FMA(KP555570233, Tc1, KP831469612 * TbY);
+		    }
+	       }
+	       {
+		    E T36, Teh, Tek, TgF, T3B, Tef, Tee, TgE, Teg, Tel;
+		    {
+			 E T2U, T35, Tei, Tej;
+			 T2U = T2O + T2T;
+			 T35 = T2Z + T34;
+			 T36 = T2U + T35;
+			 Teh = T2U - T35;
+			 Tei = T87 + T88;
+			 Tej = T82 + T83;
+			 Tek = Tei - Tej;
+			 TgF = Tei + Tej;
+		    }
+		    {
+			 E T3p, T3A, Tec, Ted;
+			 T3p = T3b + T3o;
+			 T3A = T3u + T3z;
+			 T3B = T3p + T3A;
+			 Tef = T3A - T3p;
+			 Tec = T7W + T7X;
+			 Ted = T8e + T8f;
+			 Tee = Tec - Ted;
+			 TgE = Tec + Ted;
+		    }
+		    T3C = T36 + T3B;
+		    Thy = TgE + TgF;
+		    Teg = Tee - Tef;
+		    Tel = Teh - Tek;
+		    Tem = FNMS(KP923879532, Tel, KP382683432 * Teg);
+		    Tfy = FMA(KP923879532, Teg, KP382683432 * Tel);
+		    {
+			 E TfS, TfT, TgG, TgH;
+			 TfS = Tee + Tef;
+			 TfT = Teh + Tek;
+			 TfU = FNMS(KP382683432, TfT, KP923879532 * TfS);
+			 Tgk = FMA(KP382683432, TfS, KP923879532 * TfT);
+			 TgG = TgE - TgF;
+			 TgH = T36 - T3B;
+			 TgI = TgG - TgH;
+			 Thi = TgH + TgG;
+		    }
+	       }
+	       {
+		    E T6A, Tfl, Th7, Tf4, T6e, Tar, T9Y, TcH, Tav, Tcw, T9M, Tfj;
+		    T6A = T6o + T6z;
+		    Tfl = T6z - T6o;
+		    Th7 = Tf2 + Tf3;
+		    Tf4 = Tf2 - Tf3;
+		    {
+			 E T6d, T9S, T9X, Tat, Tau, T9L;
+			 T6d = FNMS(T6b, T6c, T69 * T6a);
+			 T6e = T68 + T6d;
+			 Tar = T68 - T6d;
+			 T9S = T9Q - T9R;
+			 T9X = T9T + T9W;
+			 T9Y = KP707106781 * (T9S - T9X);
+			 TcH = KP707106781 * (T9S + T9X);
+			 Tat = T9T - T9W;
+			 Tau = T9R + T9Q;
+			 Tav = KP707106781 * (Tat - Tau);
+			 Tcw = KP707106781 * (Tau + Tat);
+			 T9L = FMA(T6b, T6a, T69 * T6c);
+			 T9M = T9K - T9L;
+			 Tfj = T9K + T9L;
+		    }
+		    {
+			 E T6f, Tfk, Th6, T9N;
+			 T6f = T65 + T6e;
+			 T6B = T6f + T6A;
+			 Th1 = T6f - T6A;
+			 Tfk = Tfi - Tfj;
+			 Tfm = Tfk - Tfl;
+			 Tga = Tfk + Tfl;
+			 Th6 = Tfi + Tfj;
+			 Th8 = Th6 - Th7;
+			 ThI = Th6 + Th7;
+			 T9N = T9J - T9M;
+			 T9Z = T9N - T9Y;
+			 Tbh = T9N + T9Y;
+		    }
+		    {
+			 E Tas, TcG, Tf1, Tcv;
+			 Tas = Taq + Tar;
+			 Taw = Tas - Tav;
+			 Tbk = Tas + Tav;
+			 TcG = Taq - Tar;
+			 TcI = TcG - TcH;
+			 Tdw = TcG + TcH;
+			 Tf1 = T65 - T6e;
+			 Tf5 = Tf1 - Tf4;
+			 Tg7 = Tf1 + Tf4;
+			 Tcv = T9J + T9M;
+			 Tcx = Tcv - Tcw;
+			 Tdt = Tcv + Tcw;
+		    }
+	       }
+	       {
+		    E T8Z, T9B, T5b, TeD, TeU, TgR, T94, T9A, T4L, T8T, T9y, TeB, T4V;
+		    T8Z = T8V - T8Y;
+		    T9B = T8V + T8Y;
+		    T4V = T4P + T4U;
+		    T5b = T4V + T5a;
+		    TeD = T5a - T4V;
+		    {
+			 E TeS, T90, T93, T4K, T9x;
+			 TeS = T91 + T92;
+			 TeU = TeS - TeT;
+			 TgR = TeS + TeT;
+			 T90 = T4P - T4U;
+			 T93 = T91 - T92;
+			 T94 = T90 + T93;
+			 T9A = T93 - T90;
+			 T4K = FMA(T4G, T4H, T4I * T4J);
+			 T4L = T4F + T4K;
+			 T8T = T4F - T4K;
+			 T9x = FNMS(T4I, T4H, T4G * T4J);
+			 T9y = T9w - T9x;
+			 TeB = T9w + T9x;
+		    }
+		    {
+			 E T4M, TeR, TgQ, TeC;
+			 T4M = T4C + T4L;
+			 T5c = T4M + T5b;
+			 TgV = T4M - T5b;
+			 TeR = T4C - T4L;
+			 TeV = TeR - TeU;
+			 Tg0 = TeR + TeU;
+			 TgQ = TeA + TeB;
+			 TgS = TgQ - TgR;
+			 ThD = TgQ + TgR;
+			 TeC = TeA - TeB;
+			 TeE = TeC - TeD;
+			 Tg3 = TeC + TeD;
+		    }
+		    {
+			 E T8U, T95, Tcc, Tcd;
+			 T8U = T8S + T8T;
+			 T95 = KP707106781 * (T8Z - T94);
+			 T96 = T8U - T95;
+			 Tbd = T8U + T95;
+			 Tcc = T8S - T8T;
+			 Tcd = KP707106781 * (T9A + T9B);
+			 Tce = Tcc - Tcd;
+			 Tdp = Tcc + Tcd;
+		    }
+		    {
+			 E Tcn, Tco, T9z, T9C;
+			 Tcn = T9v + T9y;
+			 Tco = KP707106781 * (T94 + T8Z);
+			 Tcp = Tcn - Tco;
+			 Tdm = Tcn + Tco;
+			 T9z = T9v - T9y;
+			 T9C = KP707106781 * (T9A - T9B);
+			 T9D = T9z - T9C;
+			 Tba = T9z + T9C;
+		    }
+	       }
+	       {
+		    E Tv, T7h, TdY, ThY, Ti2, Tj1, T16, Tj2, T1K, Tiw, T7q, TbK, T7v, TbL, T7k;
+		    E ThZ, T7r, T7u, T7i;
+		    {
+			 E Tu, TdW, TdX, Ti0, TM;
+			 Tu = FNMS(Ts, Tt, To * Tp);
+			 Tv = T1 + Tu;
+			 T7h = T1 - Tu;
+			 TdW = T7m + T7n;
+			 TdX = T7s + T7t;
+			 TdY = TdW - TdX;
+			 ThY = TdW + TdX;
+			 Ti0 = FMA(Ts, Tp, To * Tt);
+			 Ti2 = Ti0 + Ti1;
+			 Tj1 = Ti1 - Ti0;
+			 TM = FMA(TG, TH, TK * TL);
+			 T16 = TM + T15;
+			 Tj2 = TM - T15;
+		    }
+		    {
+			 E T1s, T1J, T7o, T7p;
+			 T1s = T1g + T1r;
+			 T1J = T1z + T1I;
+			 T1K = T1s + T1J;
+			 Tiw = T1J - T1s;
+			 T7o = T7m - T7n;
+			 T7p = T1g - T1r;
+			 T7q = T7o - T7p;
+			 TbK = T7p + T7o;
+		    }
+		    T7r = T1z - T1I;
+		    T7u = T7s - T7t;
+		    T7v = T7r + T7u;
+		    TbL = T7r - T7u;
+		    T7i = FNMS(TK, TH, TG * TL);
+		    T7k = T7i - T7j;
+		    ThZ = T7i + T7j;
+		    {
+			 E T17, Ti3, Tix, TdV;
+			 T17 = Tv + T16;
+			 T1L = T17 + T1K;
+			 Tgz = T17 - T1K;
+			 Ti3 = ThZ + Ti2;
+			 Ti4 = ThY + Ti3;
+			 Tii = Ti3 - ThY;
+			 Tix = Ti2 - ThZ;
+			 Tiy = Tiw + Tix;
+			 TiM = Tix - Tiw;
+			 TdV = Tv - T16;
+			 TdZ = TdV - TdY;
+			 TfN = TdV + TdY;
+		    }
+		    {
+			 E T7l, T7w, Tj0, Tj3;
+			 T7l = T7h - T7k;
+			 T7w = KP707106781 * (T7q - T7v);
+			 T7x = T7l - T7w;
+			 TaX = T7l + T7w;
+			 Tj0 = KP707106781 * (T7q + T7v);
+			 Tj3 = Tj1 - Tj2;
+			 Tj4 = Tj0 + Tj3;
+			 Tji = Tj3 - Tj0;
+		    }
+		    {
+			 E Tjw, Tjx, TbJ, TbM;
+			 Tjw = KP707106781 * (TbL - TbK);
+			 Tjx = Tj2 + Tj1;
+			 Tjy = Tjw + Tjx;
+			 TjM = Tjx - Tjw;
+			 TbJ = T7h + T7k;
+			 TbM = KP707106781 * (TbK + TbL);
+			 TbN = TbJ - TbM;
+			 Td9 = TbJ + TbM;
+		    }
+	       }
+	       {
+		    E T4t, ThR, Ti6, Ti8, T7g, Ti7, ThU, ThV;
+		    {
+			 E T2L, T4s, ThW, Ti5;
+			 T2L = T1L + T2K;
+			 T4s = T3C + T4r;
+			 T4t = T2L + T4s;
+			 ThR = T2L - T4s;
+			 ThW = Thy + Thz;
+			 Ti5 = ThX + Ti4;
+			 Ti6 = ThW + Ti5;
+			 Ti8 = Ti5 - ThW;
+		    }
+		    {
+			 E T5S, T7f, ThS, ThT;
+			 T5S = T5c + T5R;
+			 T7f = T6B + T7e;
+			 T7g = T5S + T7f;
+			 Ti7 = T7f - T5S;
+			 ThS = ThD + ThE;
+			 ThT = ThI + ThJ;
+			 ThU = ThS - ThT;
+			 ThV = ThS + ThT;
+		    }
+		    iio[-WS(ios, 32)] = T4t - T7g;
+		    rio[WS(ios, 32)] = ThV - Ti6;
+		    rio[0] = T4t + T7g;
+		    iio[0] = ThV + Ti6;
+		    iio[-WS(ios, 48)] = ThR - ThU;
+		    rio[WS(ios, 48)] = Ti7 - Ti8;
+		    rio[WS(ios, 16)] = ThR + ThU;
+		    iio[-WS(ios, 16)] = Ti7 + Ti8;
+	       }
+	       {
+		    E ThB, ThN, Tic, Tie, ThG, ThO, ThL, ThP;
+		    {
+			 E Thx, ThA, Tia, Tib;
+			 Thx = T1L - T2K;
+			 ThA = Thy - Thz;
+			 ThB = Thx + ThA;
+			 ThN = Thx - ThA;
+			 Tia = T4r - T3C;
+			 Tib = Ti4 - ThX;
+			 Tic = Tia + Tib;
+			 Tie = Tib - Tia;
+		    }
+		    {
+			 E ThC, ThF, ThH, ThK;
+			 ThC = T5c - T5R;
+			 ThF = ThD - ThE;
+			 ThG = ThC + ThF;
+			 ThO = ThF - ThC;
+			 ThH = T6B - T7e;
+			 ThK = ThI - ThJ;
+			 ThL = ThH - ThK;
+			 ThP = ThH + ThK;
+		    }
+		    {
+			 E ThM, Ti9, ThQ, Tid;
+			 ThM = KP707106781 * (ThG + ThL);
+			 iio[-WS(ios, 40)] = ThB - ThM;
+			 rio[WS(ios, 8)] = ThB + ThM;
+			 Ti9 = KP707106781 * (ThO + ThP);
+			 rio[WS(ios, 40)] = Ti9 - Tic;
+			 iio[-WS(ios, 8)] = Ti9 + Tic;
+			 ThQ = KP707106781 * (ThO - ThP);
+			 iio[-WS(ios, 56)] = ThN - ThQ;
+			 rio[WS(ios, 24)] = ThN + ThQ;
+			 Tid = KP707106781 * (ThL - ThG);
+			 rio[WS(ios, 56)] = Tid - Tie;
+			 iio[-WS(ios, 24)] = Tid + Tie;
+		    }
+	       }
+	       {
+		    E TgP, Thd, Tiq, Tis, Th0, The, Thb, Thf;
+		    {
+			 E TgD, TgO, Tio, Tip;
+			 TgD = Tgz - TgC;
+			 TgO = KP707106781 * (TgI - TgN);
+			 TgP = TgD + TgO;
+			 Thd = TgD - TgO;
+			 Tio = KP707106781 * (Thj - Thi);
+			 Tip = Tii - Tih;
+			 Tiq = Tio + Tip;
+			 Tis = Tip - Tio;
+		    }
+		    {
+			 E TgU, TgZ, Th5, Tha;
+			 TgU = TgS - TgT;
+			 TgZ = TgV - TgY;
+			 Th0 = FMA(KP923879532, TgU, KP382683432 * TgZ);
+			 The = FNMS(KP923879532, TgZ, KP382683432 * TgU);
+			 Th5 = Th1 - Th4;
+			 Tha = Th8 - Th9;
+			 Thb = FNMS(KP923879532, Tha, KP382683432 * Th5);
+			 Thf = FMA(KP382683432, Tha, KP923879532 * Th5);
+		    }
+		    {
+			 E Thc, Tin, Thg, Tir;
+			 Thc = Th0 + Thb;
+			 iio[-WS(ios, 44)] = TgP - Thc;
+			 rio[WS(ios, 12)] = TgP + Thc;
+			 Tin = The + Thf;
+			 rio[WS(ios, 44)] = Tin - Tiq;
+			 iio[-WS(ios, 12)] = Tin + Tiq;
+			 Thg = The - Thf;
+			 iio[-WS(ios, 60)] = Thd - Thg;
+			 rio[WS(ios, 28)] = Thd + Thg;
+			 Tir = Thb - Th0;
+			 rio[WS(ios, 60)] = Tir - Tis;
+			 iio[-WS(ios, 28)] = Tir + Tis;
+		    }
+	       }
+	       {
+		    E TfB, TfJ, TiO, TiQ, TfE, TfK, TfH, TfL;
+		    {
+			 E Tfx, TfA, TiK, TiN;
+			 Tfx = TdZ + Tea;
+			 TfA = Tfy + Tfz;
+			 TfB = Tfx + TfA;
+			 TfJ = Tfx - TfA;
+			 TiK = Tem + Tex;
+			 TiN = TiL + TiM;
+			 TiO = TiK + TiN;
+			 TiQ = TiN - TiK;
+		    }
+		    {
+			 E TfC, TfD, TfF, TfG;
+			 TfC = TeE + TeP;
+			 TfD = TeV + TeY;
+			 TfE = FMA(KP555570233, TfC, KP831469612 * TfD);
+			 TfK = FNMS(KP555570233, TfD, KP831469612 * TfC);
+			 TfF = Tf5 + Tfg;
+			 TfG = Tfm + Tfp;
+			 TfH = FNMS(KP555570233, TfG, KP831469612 * TfF);
+			 TfL = FMA(KP831469612, TfG, KP555570233 * TfF);
+		    }
+		    {
+			 E TfI, TiJ, TfM, TiP;
+			 TfI = TfE + TfH;
+			 iio[-WS(ios, 38)] = TfB - TfI;
+			 rio[WS(ios, 6)] = TfB + TfI;
+			 TiJ = TfK + TfL;
+			 rio[WS(ios, 38)] = TiJ - TiO;
+			 iio[-WS(ios, 6)] = TiJ + TiO;
+			 TfM = TfK - TfL;
+			 iio[-WS(ios, 54)] = TfJ - TfM;
+			 rio[WS(ios, 22)] = TfJ + TfM;
+			 TiP = TfH - TfE;
+			 rio[WS(ios, 54)] = TiP - TiQ;
+			 iio[-WS(ios, 22)] = TiP + TiQ;
+		    }
+	       }
+	       {
+		    E Thl, Tht, Tik, Tim, Tho, Thu, Thr, Thv;
+		    {
+			 E Thh, Thk, Tig, Tij;
+			 Thh = Tgz + TgC;
+			 Thk = KP707106781 * (Thi + Thj);
+			 Thl = Thh + Thk;
+			 Tht = Thh - Thk;
+			 Tig = KP707106781 * (TgI + TgN);
+			 Tij = Tih + Tii;
+			 Tik = Tig + Tij;
+			 Tim = Tij - Tig;
+		    }
+		    {
+			 E Thm, Thn, Thp, Thq;
+			 Thm = TgS + TgT;
+			 Thn = TgV + TgY;
+			 Tho = FMA(KP382683432, Thm, KP923879532 * Thn);
+			 Thu = FNMS(KP382683432, Thn, KP923879532 * Thm);
+			 Thp = Th1 + Th4;
+			 Thq = Th8 + Th9;
+			 Thr = FNMS(KP382683432, Thq, KP923879532 * Thp);
+			 Thv = FMA(KP923879532, Thq, KP382683432 * Thp);
+		    }
+		    {
+			 E Ths, Tif, Thw, Til;
+			 Ths = Tho + Thr;
+			 iio[-WS(ios, 36)] = Thl - Ths;
+			 rio[WS(ios, 4)] = Thl + Ths;
+			 Tif = Thu + Thv;
+			 rio[WS(ios, 36)] = Tif - Tik;
+			 iio[-WS(ios, 4)] = Tif + Tik;
+			 Thw = Thu - Thv;
+			 iio[-WS(ios, 52)] = Tht - Thw;
+			 rio[WS(ios, 20)] = Tht + Thw;
+			 Til = Thr - Tho;
+			 rio[WS(ios, 52)] = Til - Tim;
+			 iio[-WS(ios, 20)] = Til + Tim;
+		    }
+	       }
+	       {
+		    E Tez, Tft, TiU, TiW, Tf0, Tfu, Tfr, Tfv;
+		    {
+			 E Teb, Tey, TiS, TiT;
+			 Teb = TdZ - Tea;
+			 Tey = Tem - Tex;
+			 Tez = Teb + Tey;
+			 Tft = Teb - Tey;
+			 TiS = Tfz - Tfy;
+			 TiT = TiM - TiL;
+			 TiU = TiS + TiT;
+			 TiW = TiT - TiS;
+		    }
+		    {
+			 E TeQ, TeZ, Tfh, Tfq;
+			 TeQ = TeE - TeP;
+			 TeZ = TeV - TeY;
+			 Tf0 = FMA(KP980785280, TeQ, KP195090322 * TeZ);
+			 Tfu = FNMS(KP980785280, TeZ, KP195090322 * TeQ);
+			 Tfh = Tf5 - Tfg;
+			 Tfq = Tfm - Tfp;
+			 Tfr = FNMS(KP980785280, Tfq, KP195090322 * Tfh);
+			 Tfv = FMA(KP195090322, Tfq, KP980785280 * Tfh);
+		    }
+		    {
+			 E Tfs, TiR, Tfw, TiV;
+			 Tfs = Tf0 + Tfr;
+			 iio[-WS(ios, 46)] = Tez - Tfs;
+			 rio[WS(ios, 14)] = Tez + Tfs;
+			 TiR = Tfu + Tfv;
+			 rio[WS(ios, 46)] = TiR - TiU;
+			 iio[-WS(ios, 14)] = TiR + TiU;
+			 Tfw = Tfu - Tfv;
+			 iio[-WS(ios, 62)] = Tft - Tfw;
+			 rio[WS(ios, 30)] = Tft + Tfw;
+			 TiV = Tfr - Tf0;
+			 rio[WS(ios, 62)] = TiV - TiW;
+			 iio[-WS(ios, 30)] = TiV + TiW;
+		    }
+	       }
+	       {
+		    E TfZ, Tgf, TiG, TiI, Tg6, Tgg, Tgd, Tgh;
+		    {
+			 E TfR, TfY, TiE, TiF;
+			 TfR = TfN - TfQ;
+			 TfY = TfU - TfX;
+			 TfZ = TfR + TfY;
+			 Tgf = TfR - TfY;
+			 TiE = Tgl - Tgk;
+			 TiF = Tiy - Tiv;
+			 TiG = TiE + TiF;
+			 TiI = TiF - TiE;
+		    }
+		    {
+			 E Tg2, Tg5, Tg9, Tgc;
+			 Tg2 = Tg0 - Tg1;
+			 Tg5 = Tg3 - Tg4;
+			 Tg6 = FMA(KP555570233, Tg2, KP831469612 * Tg5);
+			 Tgg = FNMS(KP831469612, Tg2, KP555570233 * Tg5);
+			 Tg9 = Tg7 - Tg8;
+			 Tgc = Tga - Tgb;
+			 Tgd = FNMS(KP831469612, Tgc, KP555570233 * Tg9);
+			 Tgh = FMA(KP831469612, Tg9, KP555570233 * Tgc);
+		    }
+		    {
+			 E Tge, TiD, Tgi, TiH;
+			 Tge = Tg6 + Tgd;
+			 iio[-WS(ios, 42)] = TfZ - Tge;
+			 rio[WS(ios, 10)] = TfZ + Tge;
+			 TiD = Tgg + Tgh;
+			 rio[WS(ios, 42)] = TiD - TiG;
+			 iio[-WS(ios, 10)] = TiD + TiG;
+			 Tgi = Tgg - Tgh;
+			 iio[-WS(ios, 58)] = Tgf - Tgi;
+			 rio[WS(ios, 26)] = Tgf + Tgi;
+			 TiH = Tgd - Tg6;
+			 rio[WS(ios, 58)] = TiH - TiI;
+			 iio[-WS(ios, 26)] = TiH + TiI;
+		    }
+	       }
+	       {
+		    E Tgn, Tgv, TiA, TiC, Tgq, Tgw, Tgt, Tgx;
+		    {
+			 E Tgj, Tgm, Tiu, Tiz;
+			 Tgj = TfN + TfQ;
+			 Tgm = Tgk + Tgl;
+			 Tgn = Tgj + Tgm;
+			 Tgv = Tgj - Tgm;
+			 Tiu = TfU + TfX;
+			 Tiz = Tiv + Tiy;
+			 TiA = Tiu + Tiz;
+			 TiC = Tiz - Tiu;
+		    }
+		    {
+			 E Tgo, Tgp, Tgr, Tgs;
+			 Tgo = Tg0 + Tg1;
+			 Tgp = Tg3 + Tg4;
+			 Tgq = FMA(KP980785280, Tgo, KP195090322 * Tgp);
+			 Tgw = FNMS(KP195090322, Tgo, KP980785280 * Tgp);
+			 Tgr = Tg7 + Tg8;
+			 Tgs = Tga + Tgb;
+			 Tgt = FNMS(KP195090322, Tgs, KP980785280 * Tgr);
+			 Tgx = FMA(KP195090322, Tgr, KP980785280 * Tgs);
+		    }
+		    {
+			 E Tgu, Tit, Tgy, TiB;
+			 Tgu = Tgq + Tgt;
+			 iio[-WS(ios, 34)] = Tgn - Tgu;
+			 rio[WS(ios, 2)] = Tgn + Tgu;
+			 Tit = Tgw + Tgx;
+			 rio[WS(ios, 34)] = Tit - TiA;
+			 iio[-WS(ios, 2)] = Tit + TiA;
+			 Tgy = Tgw - Tgx;
+			 iio[-WS(ios, 50)] = Tgv - Tgy;
+			 rio[WS(ios, 18)] = Tgv + Tgy;
+			 TiB = Tgt - Tgq;
+			 rio[WS(ios, 50)] = TiB - TiC;
+			 iio[-WS(ios, 18)] = TiB + TiC;
+		    }
+	       }
+	       {
+		    E T7V, TjN, TjT, TaH, T8O, TjK, TaK, TjS, TaO, TaU, T9I, TaE, TaR, TaV, TaB;
+		    E TaF, T8N;
+		    T7V = T7x - T7U;
+		    TjN = TjL + TjM;
+		    TjT = TjM - TjL;
+		    TaH = T7x + T7U;
+		    T8N = FMA(KP195090322, T8D, KP980785280 * T8M);
+		    T8O = T8m - T8N;
+		    TjK = T8m + T8N;
+		    {
+			 E TaJ, TaM, TaN, T9u, T9H;
+			 TaJ = FNMS(KP980785280, T8D, KP195090322 * T8M);
+			 TaK = TaI + TaJ;
+			 TjS = TaJ - TaI;
+			 TaM = T96 + T9t;
+			 TaN = T9D + T9G;
+			 TaO = FMA(KP634393284, TaM, KP773010453 * TaN);
+			 TaU = FNMS(KP634393284, TaN, KP773010453 * TaM);
+			 T9u = T96 - T9t;
+			 T9H = T9D - T9G;
+			 T9I = FMA(KP995184726, T9u, KP098017140 * T9H);
+			 TaE = FNMS(KP995184726, T9H, KP098017140 * T9u);
+			 {
+			      E TaP, TaQ, Tan, TaA;
+			      TaP = T9Z + Tam;
+			      TaQ = Taw + Taz;
+			      TaR = FNMS(KP634393284, TaQ, KP773010453 * TaP);
+			      TaV = FMA(KP773010453, TaQ, KP634393284 * TaP);
+			      Tan = T9Z - Tam;
+			      TaA = Taw - Taz;
+			      TaB = FNMS(KP995184726, TaA, KP098017140 * Tan);
+			      TaF = FMA(KP098017140, TaA, KP995184726 * Tan);
+			 }
+		    }
+		    {
+			 E T8P, TaC, TjR, TjU;
+			 T8P = T7V + T8O;
+			 TaC = T9I + TaB;
+			 iio[-WS(ios, 47)] = T8P - TaC;
+			 rio[WS(ios, 15)] = T8P + TaC;
+			 TjR = TaE + TaF;
+			 TjU = TjS + TjT;
+			 rio[WS(ios, 47)] = TjR - TjU;
+			 iio[-WS(ios, 15)] = TjR + TjU;
+		    }
+		    {
+			 E TaD, TaG, TjV, TjW;
+			 TaD = T7V - T8O;
+			 TaG = TaE - TaF;
+			 iio[-WS(ios, 63)] = TaD - TaG;
+			 rio[WS(ios, 31)] = TaD + TaG;
+			 TjV = TaB - T9I;
+			 TjW = TjT - TjS;
+			 rio[WS(ios, 63)] = TjV - TjW;
+			 iio[-WS(ios, 31)] = TjV + TjW;
+		    }
+		    {
+			 E TaL, TaS, TjJ, TjO;
+			 TaL = TaH + TaK;
+			 TaS = TaO + TaR;
+			 iio[-WS(ios, 39)] = TaL - TaS;
+			 rio[WS(ios, 7)] = TaL + TaS;
+			 TjJ = TaU + TaV;
+			 TjO = TjK + TjN;
+			 rio[WS(ios, 39)] = TjJ - TjO;
+			 iio[-WS(ios, 7)] = TjJ + TjO;
+		    }
+		    {
+			 E TaT, TaW, TjP, TjQ;
+			 TaT = TaH - TaK;
+			 TaW = TaU - TaV;
+			 iio[-WS(ios, 55)] = TaT - TaW;
+			 rio[WS(ios, 23)] = TaT + TaW;
+			 TjP = TaR - TaO;
+			 TjQ = TjN - TjK;
+			 rio[WS(ios, 55)] = TjP - TjQ;
+			 iio[-WS(ios, 23)] = TjP + TjQ;
+		    }
+	       }
+	       {
+		    E TbV, Tjj, Tjp, TcT, Tca, Tjg, TcW, Tjo, Td0, Td6, Tcu, TcQ, Td3, Td7, TcN;
+		    E TcR, Tc9;
+		    TbV = TbN - TbU;
+		    Tjj = Tjh + Tji;
+		    Tjp = Tji - Tjh;
+		    TcT = TbN + TbU;
+		    Tc9 = FMA(KP831469612, Tc5, KP555570233 * Tc8);
+		    Tca = Tc2 - Tc9;
+		    Tjg = Tc2 + Tc9;
+		    {
+			 E TcV, TcY, TcZ, Tcm, Tct;
+			 TcV = FNMS(KP831469612, Tc8, KP555570233 * Tc5);
+			 TcW = TcU + TcV;
+			 Tjo = TcV - TcU;
+			 TcY = Tce + Tcl;
+			 TcZ = Tcp + Tcs;
+			 Td0 = FMA(KP471396736, TcY, KP881921264 * TcZ);
+			 Td6 = FNMS(KP471396736, TcZ, KP881921264 * TcY);
+			 Tcm = Tce - Tcl;
+			 Tct = Tcp - Tcs;
+			 Tcu = FMA(KP956940335, Tcm, KP290284677 * Tct);
+			 TcQ = FNMS(KP956940335, Tct, KP290284677 * Tcm);
+			 {
+			      E Td1, Td2, TcF, TcM;
+			      Td1 = Tcx + TcE;
+			      Td2 = TcI + TcL;
+			      Td3 = FNMS(KP471396736, Td2, KP881921264 * Td1);
+			      Td7 = FMA(KP881921264, Td2, KP471396736 * Td1);
+			      TcF = Tcx - TcE;
+			      TcM = TcI - TcL;
+			      TcN = FNMS(KP956940335, TcM, KP290284677 * TcF);
+			      TcR = FMA(KP290284677, TcM, KP956940335 * TcF);
+			 }
+		    }
+		    {
+			 E Tcb, TcO, Tjn, Tjq;
+			 Tcb = TbV + Tca;
+			 TcO = Tcu + TcN;
+			 iio[-WS(ios, 45)] = Tcb - TcO;
+			 rio[WS(ios, 13)] = Tcb + TcO;
+			 Tjn = TcQ + TcR;
+			 Tjq = Tjo + Tjp;
+			 rio[WS(ios, 45)] = Tjn - Tjq;
+			 iio[-WS(ios, 13)] = Tjn + Tjq;
+		    }
+		    {
+			 E TcP, TcS, Tjr, Tjs;
+			 TcP = TbV - Tca;
+			 TcS = TcQ - TcR;
+			 iio[-WS(ios, 61)] = TcP - TcS;
+			 rio[WS(ios, 29)] = TcP + TcS;
+			 Tjr = TcN - Tcu;
+			 Tjs = Tjp - Tjo;
+			 rio[WS(ios, 61)] = Tjr - Tjs;
+			 iio[-WS(ios, 29)] = Tjr + Tjs;
+		    }
+		    {
+			 E TcX, Td4, Tjf, Tjk;
+			 TcX = TcT + TcW;
+			 Td4 = Td0 + Td3;
+			 iio[-WS(ios, 37)] = TcX - Td4;
+			 rio[WS(ios, 5)] = TcX + Td4;
+			 Tjf = Td6 + Td7;
+			 Tjk = Tjg + Tjj;
+			 rio[WS(ios, 37)] = Tjf - Tjk;
+			 iio[-WS(ios, 5)] = Tjf + Tjk;
+		    }
+		    {
+			 E Td5, Td8, Tjl, Tjm;
+			 Td5 = TcT - TcW;
+			 Td8 = Td6 - Td7;
+			 iio[-WS(ios, 53)] = Td5 - Td8;
+			 rio[WS(ios, 21)] = Td5 + Td8;
+			 Tjl = Td3 - Td0;
+			 Tjm = Tjj - Tjg;
+			 rio[WS(ios, 53)] = Tjl - Tjm;
+			 iio[-WS(ios, 21)] = Tjl + Tjm;
+		    }
+	       }
+	       {
+		    E Tb1, Tjz, TjF, Tbt, Tb8, Tju, Tbw, TjE, TbA, TbG, Tbg, Tbq, TbD, TbH, Tbn;
+		    E Tbr, Tb7;
+		    Tb1 = TaX - Tb0;
+		    Tjz = Tjv + Tjy;
+		    TjF = Tjy - Tjv;
+		    Tbt = TaX + Tb0;
+		    Tb7 = FMA(KP831469612, Tb5, KP555570233 * Tb6);
+		    Tb8 = Tb4 - Tb7;
+		    Tju = Tb4 + Tb7;
+		    {
+			 E Tbv, Tby, Tbz, Tbc, Tbf;
+			 Tbv = FNMS(KP555570233, Tb5, KP831469612 * Tb6);
+			 Tbw = Tbu + Tbv;
+			 TjE = Tbv - Tbu;
+			 Tby = Tba + Tbb;
+			 Tbz = Tbd + Tbe;
+			 TbA = FMA(KP956940335, Tby, KP290284677 * Tbz);
+			 TbG = FNMS(KP290284677, Tby, KP956940335 * Tbz);
+			 Tbc = Tba - Tbb;
+			 Tbf = Tbd - Tbe;
+			 Tbg = FMA(KP471396736, Tbc, KP881921264 * Tbf);
+			 Tbq = FNMS(KP881921264, Tbc, KP471396736 * Tbf);
+			 {
+			      E TbB, TbC, Tbj, Tbm;
+			      TbB = Tbh + Tbi;
+			      TbC = Tbk + Tbl;
+			      TbD = FNMS(KP290284677, TbC, KP956940335 * TbB);
+			      TbH = FMA(KP290284677, TbB, KP956940335 * TbC);
+			      Tbj = Tbh - Tbi;
+			      Tbm = Tbk - Tbl;
+			      Tbn = FNMS(KP881921264, Tbm, KP471396736 * Tbj);
+			      Tbr = FMA(KP881921264, Tbj, KP471396736 * Tbm);
+			 }
+		    }
+		    {
+			 E Tb9, Tbo, TjD, TjG;
+			 Tb9 = Tb1 + Tb8;
+			 Tbo = Tbg + Tbn;
+			 iio[-WS(ios, 43)] = Tb9 - Tbo;
+			 rio[WS(ios, 11)] = Tb9 + Tbo;
+			 TjD = Tbq + Tbr;
+			 TjG = TjE + TjF;
+			 rio[WS(ios, 43)] = TjD - TjG;
+			 iio[-WS(ios, 11)] = TjD + TjG;
+		    }
+		    {
+			 E Tbp, Tbs, TjH, TjI;
+			 Tbp = Tb1 - Tb8;
+			 Tbs = Tbq - Tbr;
+			 iio[-WS(ios, 59)] = Tbp - Tbs;
+			 rio[WS(ios, 27)] = Tbp + Tbs;
+			 TjH = Tbn - Tbg;
+			 TjI = TjF - TjE;
+			 rio[WS(ios, 59)] = TjH - TjI;
+			 iio[-WS(ios, 27)] = TjH + TjI;
+		    }
+		    {
+			 E Tbx, TbE, Tjt, TjA;
+			 Tbx = Tbt + Tbw;
+			 TbE = TbA + TbD;
+			 iio[-WS(ios, 35)] = Tbx - TbE;
+			 rio[WS(ios, 3)] = Tbx + TbE;
+			 Tjt = TbG + TbH;
+			 TjA = Tju + Tjz;
+			 rio[WS(ios, 35)] = Tjt - TjA;
+			 iio[-WS(ios, 3)] = Tjt + TjA;
+		    }
+		    {
+			 E TbF, TbI, TjB, TjC;
+			 TbF = Tbt - Tbw;
+			 TbI = TbG - TbH;
+			 iio[-WS(ios, 51)] = TbF - TbI;
+			 rio[WS(ios, 19)] = TbF + TbI;
+			 TjB = TbD - TbA;
+			 TjC = Tjz - Tju;
+			 rio[WS(ios, 51)] = TjB - TjC;
+			 iio[-WS(ios, 19)] = TjB + TjC;
+		    }
+	       }
+	       {
+		    E Tdd, Tj5, Tjb, TdF, Tdk, TiY, TdI, Tja, TdM, TdS, Tds, TdC, TdP, TdT, Tdz;
+		    E TdD, Tdj;
+		    Tdd = Td9 - Tdc;
+		    Tj5 = TiZ + Tj4;
+		    Tjb = Tj4 - TiZ;
+		    TdF = Td9 + Tdc;
+		    Tdj = FMA(KP195090322, Tdh, KP980785280 * Tdi);
+		    Tdk = Tdg - Tdj;
+		    TiY = Tdg + Tdj;
+		    {
+			 E TdH, TdK, TdL, Tdo, Tdr;
+			 TdH = FNMS(KP195090322, Tdi, KP980785280 * Tdh);
+			 TdI = TdG + TdH;
+			 Tja = TdH - TdG;
+			 TdK = Tdm + Tdn;
+			 TdL = Tdp + Tdq;
+			 TdM = FMA(KP995184726, TdK, KP098017140 * TdL);
+			 TdS = FNMS(KP098017140, TdK, KP995184726 * TdL);
+			 Tdo = Tdm - Tdn;
+			 Tdr = Tdp - Tdq;
+			 Tds = FMA(KP634393284, Tdo, KP773010453 * Tdr);
+			 TdC = FNMS(KP773010453, Tdo, KP634393284 * Tdr);
+			 {
+			      E TdN, TdO, Tdv, Tdy;
+			      TdN = Tdt + Tdu;
+			      TdO = Tdw + Tdx;
+			      TdP = FNMS(KP098017140, TdO, KP995184726 * TdN);
+			      TdT = FMA(KP098017140, TdN, KP995184726 * TdO);
+			      Tdv = Tdt - Tdu;
+			      Tdy = Tdw - Tdx;
+			      Tdz = FNMS(KP773010453, Tdy, KP634393284 * Tdv);
+			      TdD = FMA(KP773010453, Tdv, KP634393284 * Tdy);
+			 }
+		    }
+		    {
+			 E Tdl, TdA, Tj9, Tjc;
+			 Tdl = Tdd + Tdk;
+			 TdA = Tds + Tdz;
+			 iio[-WS(ios, 41)] = Tdl - TdA;
+			 rio[WS(ios, 9)] = Tdl + TdA;
+			 Tj9 = TdC + TdD;
+			 Tjc = Tja + Tjb;
+			 rio[WS(ios, 41)] = Tj9 - Tjc;
+			 iio[-WS(ios, 9)] = Tj9 + Tjc;
+		    }
+		    {
+			 E TdB, TdE, Tjd, Tje;
+			 TdB = Tdd - Tdk;
+			 TdE = TdC - TdD;
+			 iio[-WS(ios, 57)] = TdB - TdE;
+			 rio[WS(ios, 25)] = TdB + TdE;
+			 Tjd = Tdz - Tds;
+			 Tje = Tjb - Tja;
+			 rio[WS(ios, 57)] = Tjd - Tje;
+			 iio[-WS(ios, 25)] = Tjd + Tje;
+		    }
+		    {
+			 E TdJ, TdQ, TiX, Tj6;
+			 TdJ = TdF + TdI;
+			 TdQ = TdM + TdP;
+			 iio[-WS(ios, 33)] = TdJ - TdQ;
+			 rio[WS(ios, 1)] = TdJ + TdQ;
+			 TiX = TdS + TdT;
+			 Tj6 = TiY + Tj5;
+			 rio[WS(ios, 33)] = TiX - Tj6;
+			 iio[-WS(ios, 1)] = TiX + Tj6;
+		    }
+		    {
+			 E TdR, TdU, Tj7, Tj8;
+			 TdR = TdF - TdI;
+			 TdU = TdS - TdT;
+			 iio[-WS(ios, 49)] = TdR - TdU;
+			 rio[WS(ios, 17)] = TdR + TdU;
+			 Tj7 = TdP - TdM;
+			 Tj8 = Tj5 - TiY;
+			 rio[WS(ios, 49)] = Tj7 - Tj8;
+			 iio[-WS(ios, 17)] = Tj7 + Tj8;
+		    }
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_COS, 0, 1},
+     {TW_SIN, 0, 1},
+     {TW_COS, 0, 3},
+     {TW_SIN, 0, 3},
+     {TW_COS, 0, 9},
+     {TW_SIN, 0, 9},
+     {TW_COS, 0, 27},
+     {TW_SIN, 0, 27},
+     {TW_COS, 0, 63},
+     {TW_SIN, 0, 63},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 64, "hf2_64", twinstr, {880, 386, 274, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf2_64) (planner *p) {
+     X(khc2hc_dit_register) (p, hf2_64, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf2_8.c b/src/fftw3/rdft/codelets/r2hc/hf2_8.c
new file mode 100644
index 0000000..663e8dd
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf2_8.c
@@ -0,0 +1,192 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:27 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -twiddle-log3 -n 8 -dit -name hf2_8 -include hf.h */
+
+/*
+ * This function contains 74 FP additions, 44 FP multiplications,
+ * (or, 56 additions, 26 multiplications, 18 fused multiply/add),
+ * 50 stack variables, and 32 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf2_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf2_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf2_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf2_8(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 6) {
+	  E T1, T1c, TJ, T17, TY, TV, TR, Tk, Tr, TN, TM, Tw, TB, TS, Te;
+	  E T1b;
+	  T1 = rio[0];
+	  T1c = iio[-WS(ios, 7)];
+	  {
+	       E T9, T3, T6, T4, Tb, T7, Ta, Tg, Ti, TI, TX, Tl, Tm, Tn, TW;
+	       E Tp, TF, Td, TA, Th, Ty, Tj, Tv, To, Tq, Tu;
+	       T9 = rio[WS(ios, 4)];
+	       {
+		    E TD, TH, TE, TG, T2, T5;
+		    TD = rio[WS(ios, 7)];
+		    TH = iio[-WS(ios, 4)];
+		    TE = iio[0];
+		    TG = rio[WS(ios, 3)];
+		    T2 = W[2];
+		    T5 = W[3];
+		    T3 = W[0];
+		    T6 = W[1];
+		    T4 = T2 * T3;
+		    Tb = T5 * T3;
+		    T7 = T5 * T6;
+		    Ta = T2 * T6;
+		    Tg = T4 + T7;
+		    Ti = Ta - Tb;
+		    TI = FMA(T2, TG, T5 * TH);
+		    TX = FNMS(T5, TG, T2 * TH);
+		    Tl = W[4];
+		    Tm = W[5];
+		    Tn = FMA(Tl, T3, Tm * T6);
+		    TW = FNMS(Tm, TD, Tl * TE);
+		    Tp = FNMS(Tm, T3, Tl * T6);
+		    TF = FMA(Tl, TD, Tm * TE);
+	       }
+	       Td = iio[-WS(ios, 3)];
+	       TA = iio[-WS(ios, 2)];
+	       Th = rio[WS(ios, 2)];
+	       Ty = rio[WS(ios, 5)];
+	       Tj = iio[-WS(ios, 5)];
+	       Tv = iio[-WS(ios, 6)];
+	       To = rio[WS(ios, 6)];
+	       Tq = iio[-WS(ios, 1)];
+	       Tu = rio[WS(ios, 1)];
+	       TJ = TF + TI;
+	       T17 = TW + TX;
+	       TY = TW - TX;
+	       TV = TF - TI;
+	       TR = FNMS(T6, Tu, T3 * Tv);
+	       Tk = FNMS(Ti, Tj, Tg * Th);
+	       Tr = FNMS(Tp, Tq, Tn * To);
+	       TN = FMA(Tp, To, Tn * Tq);
+	       TM = FMA(Ti, Th, Tg * Tj);
+	       Tw = FMA(T3, Tu, T6 * Tv);
+	       {
+		    E Tx, Tz, T8, Tc;
+		    Tx = FNMS(Tm, Ti, Tl * Tg);
+		    Tz = FMA(Tl, Ti, Tm * Tg);
+		    TB = FMA(Tx, Ty, Tz * TA);
+		    TS = FNMS(Tz, Ty, Tx * TA);
+		    T8 = T4 - T7;
+		    Tc = Ta + Tb;
+		    Te = FMA(T8, T9, Tc * Td);
+		    T1b = FNMS(Tc, T9, T8 * Td);
+	       }
+	  }
+	  {
+	       E TK, T1f, T18, T19, Tt, T15, T1e, T1g, TC, T16;
+	       TC = Tw + TB;
+	       TK = TC + TJ;
+	       T1f = TJ - TC;
+	       T16 = TR + TS;
+	       T18 = T16 - T17;
+	       T19 = T16 + T17;
+	       {
+		    E Tf, Ts, T1a, T1d;
+		    Tf = T1 + Te;
+		    Ts = Tk + Tr;
+		    Tt = Tf + Ts;
+		    T15 = Tf - Ts;
+		    T1a = TM + TN;
+		    T1d = T1b + T1c;
+		    T1e = T1a + T1d;
+		    T1g = T1d - T1a;
+	       }
+	       iio[-WS(ios, 4)] = Tt - TK;
+	       rio[WS(ios, 4)] = T19 - T1e;
+	       rio[0] = Tt + TK;
+	       iio[0] = T19 + T1e;
+	       iio[-WS(ios, 6)] = T15 - T18;
+	       rio[WS(ios, 6)] = T1f - T1g;
+	       rio[WS(ios, 2)] = T15 + T18;
+	       iio[-WS(ios, 2)] = T1f + T1g;
+	  }
+	  {
+	       E TZ, T13, TP, T11, TU, T12, T1k, T1m, TL, TO;
+	       TZ = TV - TY;
+	       T13 = TV + TY;
+	       TL = T1 - Te;
+	       TO = TM - TN;
+	       TP = TL + TO;
+	       T11 = TL - TO;
+	       {
+		    E TQ, TT, T1i, T1j;
+		    TQ = Tw - TB;
+		    TT = TR - TS;
+		    TU = TQ + TT;
+		    T12 = TT - TQ;
+		    T1i = T1c - T1b;
+		    T1j = Tk - Tr;
+		    T1k = T1i - T1j;
+		    T1m = T1j + T1i;
+	       }
+	       {
+		    E T10, T1h, T14, T1l;
+		    T10 = KP707106781 * (TU + TZ);
+		    iio[-WS(ios, 5)] = TP - T10;
+		    rio[WS(ios, 1)] = TP + T10;
+		    T1h = KP707106781 * (T12 + T13);
+		    rio[WS(ios, 5)] = T1h - T1k;
+		    iio[-WS(ios, 1)] = T1h + T1k;
+		    T14 = KP707106781 * (T12 - T13);
+		    iio[-WS(ios, 7)] = T11 - T14;
+		    rio[WS(ios, 3)] = T11 + T14;
+		    T1l = KP707106781 * (TZ - TU);
+		    rio[WS(ios, 7)] = T1l - T1m;
+		    iio[-WS(ios, 3)] = T1l + T1m;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_COS, 0, 1},
+     {TW_SIN, 0, 1},
+     {TW_COS, 0, 3},
+     {TW_SIN, 0, 3},
+     {TW_COS, 0, 7},
+     {TW_SIN, 0, 7},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 8, "hf2_8", twinstr, {56, 26, 18, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf2_8) (planner *p) {
+     X(khc2hc_dit_register) (p, hf2_8, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_10.c b/src/fftw3/rdft/codelets/r2hc/hf_10.c
new file mode 100644
index 0000000..bb970fa
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_10.c
@@ -0,0 +1,253 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:02 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 10 -dit -name hf_10 -include hf.h */
+
+/*
+ * This function contains 102 FP additions, 60 FP multiplications,
+ * (or, 72 additions, 30 multiplications, 30 fused multiply/add),
+ * 45 stack variables, and 40 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_10.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_10.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_10.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_10(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 18) {
+	  E T7, T1O, TT, T1C, TF, TQ, TR, T1r, T1s, T1L, TX, TY, TZ, T16, T19;
+	  E T1y, Ti, Tt, Tu, T1o, T1p, T1M, TU, TV, TW, T1d, T1g, T1x;
+	  {
+	       E T1, T1B, T6, T1A;
+	       T1 = rio[0];
+	       T1B = iio[-WS(ios, 9)];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = rio[WS(ios, 5)];
+		    T5 = iio[-WS(ios, 4)];
+		    T2 = W[8];
+		    T4 = W[9];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T1A = FNMS(T4, T3, T2 * T5);
+	       }
+	       T7 = T1 - T6;
+	       T1O = T1B - T1A;
+	       TT = T1 + T6;
+	       T1C = T1A + T1B;
+	  }
+	  {
+	       E Tz, T14, TP, T18, TE, T15, TK, T17;
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = rio[WS(ios, 4)];
+		    Ty = iio[-WS(ios, 5)];
+		    Tv = W[6];
+		    Tx = W[7];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T14 = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E TM, TO, TL, TN;
+		    TM = rio[WS(ios, 1)];
+		    TO = iio[-WS(ios, 8)];
+		    TL = W[0];
+		    TN = W[1];
+		    TP = FMA(TL, TM, TN * TO);
+		    T18 = FNMS(TN, TM, TL * TO);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = rio[WS(ios, 9)];
+		    TD = iio[0];
+		    TA = W[16];
+		    TC = W[17];
+		    TE = FMA(TA, TB, TC * TD);
+		    T15 = FNMS(TC, TB, TA * TD);
+	       }
+	       {
+		    E TH, TJ, TG, TI;
+		    TH = rio[WS(ios, 6)];
+		    TJ = iio[-WS(ios, 3)];
+		    TG = W[10];
+		    TI = W[11];
+		    TK = FMA(TG, TH, TI * TJ);
+		    T17 = FNMS(TI, TH, TG * TJ);
+	       }
+	       TF = Tz - TE;
+	       TQ = TK - TP;
+	       TR = TF + TQ;
+	       T1r = T14 - T15;
+	       T1s = T18 - T17;
+	       T1L = T1s - T1r;
+	       TX = Tz + TE;
+	       TY = TK + TP;
+	       TZ = TX + TY;
+	       T16 = T14 + T15;
+	       T19 = T17 + T18;
+	       T1y = T16 + T19;
+	  }
+	  {
+	       E Tc, T1b, Th, T1c, Tn, T1e, Ts, T1f;
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = rio[WS(ios, 2)];
+		    Tb = iio[-WS(ios, 7)];
+		    T8 = W[2];
+		    Ta = W[3];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    T1b = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = rio[WS(ios, 7)];
+		    Tg = iio[-WS(ios, 2)];
+		    Td = W[12];
+		    Tf = W[13];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    T1c = FNMS(Tf, Te, Td * Tg);
+	       }
+	       {
+		    E Tk, Tm, Tj, Tl;
+		    Tk = rio[WS(ios, 8)];
+		    Tm = iio[-WS(ios, 1)];
+		    Tj = W[14];
+		    Tl = W[15];
+		    Tn = FMA(Tj, Tk, Tl * Tm);
+		    T1e = FNMS(Tl, Tk, Tj * Tm);
+	       }
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = rio[WS(ios, 3)];
+		    Tr = iio[-WS(ios, 6)];
+		    To = W[4];
+		    Tq = W[5];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    T1f = FNMS(Tq, Tp, To * Tr);
+	       }
+	       Ti = Tc - Th;
+	       Tt = Tn - Ts;
+	       Tu = Ti + Tt;
+	       T1o = T1b - T1c;
+	       T1p = T1e - T1f;
+	       T1M = T1o + T1p;
+	       TU = Tc + Th;
+	       TV = Tn + Ts;
+	       TW = TU + TV;
+	       T1d = T1b + T1c;
+	       T1g = T1e + T1f;
+	       T1x = T1d + T1g;
+	  }
+	  {
+	       E T1l, TS, T1m, T1u, T1w, T1q, T1t, T1v, T1n;
+	       T1l = KP559016994 * (Tu - TR);
+	       TS = Tu + TR;
+	       T1m = FNMS(KP250000000, TS, T7);
+	       T1q = T1o - T1p;
+	       T1t = T1r + T1s;
+	       T1u = FMA(KP951056516, T1q, KP587785252 * T1t);
+	       T1w = FNMS(KP587785252, T1q, KP951056516 * T1t);
+	       iio[-WS(ios, 5)] = T7 + TS;
+	       T1v = T1m - T1l;
+	       iio[-WS(ios, 7)] = T1v - T1w;
+	       rio[WS(ios, 3)] = T1v + T1w;
+	       T1n = T1l + T1m;
+	       iio[-WS(ios, 9)] = T1n - T1u;
+	       rio[WS(ios, 1)] = T1n + T1u;
+	  }
+	  {
+	       E T1S, T1N, T1T, T1R, T1V, T1P, T1Q, T1W, T1U;
+	       T1S = KP559016994 * (T1M + T1L);
+	       T1N = T1L - T1M;
+	       T1T = FMA(KP250000000, T1N, T1O);
+	       T1P = TQ - TF;
+	       T1Q = Ti - Tt;
+	       T1R = FNMS(KP951056516, T1Q, KP587785252 * T1P);
+	       T1V = FMA(KP587785252, T1Q, KP951056516 * T1P);
+	       rio[WS(ios, 5)] = T1N - T1O;
+	       T1W = T1T - T1S;
+	       rio[WS(ios, 7)] = T1V - T1W;
+	       iio[-WS(ios, 3)] = T1V + T1W;
+	       T1U = T1S + T1T;
+	       rio[WS(ios, 9)] = T1R - T1U;
+	       iio[-WS(ios, 1)] = T1R + T1U;
+	  }
+	  {
+	       E T12, T10, T11, T1i, T1k, T1a, T1h, T1j, T13;
+	       T12 = KP559016994 * (TW - TZ);
+	       T10 = TW + TZ;
+	       T11 = FNMS(KP250000000, T10, TT);
+	       T1a = T16 - T19;
+	       T1h = T1d - T1g;
+	       T1i = FNMS(KP587785252, T1h, KP951056516 * T1a);
+	       T1k = FMA(KP951056516, T1h, KP587785252 * T1a);
+	       rio[0] = TT + T10;
+	       T1j = T12 + T11;
+	       rio[WS(ios, 4)] = T1j - T1k;
+	       iio[-WS(ios, 6)] = T1j + T1k;
+	       T13 = T11 - T12;
+	       rio[WS(ios, 2)] = T13 - T1i;
+	       iio[-WS(ios, 8)] = T13 + T1i;
+	  }
+	  {
+	       E T1H, T1z, T1G, T1F, T1J, T1D, T1E, T1K, T1I;
+	       T1H = KP559016994 * (T1x - T1y);
+	       T1z = T1x + T1y;
+	       T1G = FNMS(KP250000000, T1z, T1C);
+	       T1D = TX - TY;
+	       T1E = TU - TV;
+	       T1F = FNMS(KP587785252, T1E, KP951056516 * T1D);
+	       T1J = FMA(KP951056516, T1E, KP587785252 * T1D);
+	       iio[0] = T1z + T1C;
+	       T1K = T1H + T1G;
+	       rio[WS(ios, 6)] = T1J - T1K;
+	       iio[-WS(ios, 4)] = T1J + T1K;
+	       T1I = T1G - T1H;
+	       rio[WS(ios, 8)] = T1F - T1I;
+	       iio[-WS(ios, 2)] = T1F + T1I;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 10},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 10, "hf_10", twinstr, {72, 30, 30, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_10) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_10, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_12.c b/src/fftw3/rdft/codelets/r2hc/hf_12.c
new file mode 100644
index 0000000..e399159
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_12.c
@@ -0,0 +1,300 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:06 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 12 -dit -name hf_12 -include hf.h */
+
+/*
+ * This function contains 118 FP additions, 60 FP multiplications,
+ * (or, 88 additions, 30 multiplications, 30 fused multiply/add),
+ * 47 stack variables, and 48 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_12.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_12.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_12.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_12(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 22) {
+	  E T1, T1W, T18, T22, Tc, T15, T1V, T23, TA, T1y, T1j, T1B, TL, T1g, T1z;
+	  E T1A, Ti, T1S, T1d, T25, Tt, T1a, T1T, T26, TR, T1E, T1o, T1D, T12, T1l;
+	  E T1F, T1G;
+	  {
+	       E T6, T16, Tb, T17;
+	       T1 = rio[0];
+	       T1W = iio[-WS(ios, 11)];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = rio[WS(ios, 4)];
+		    T5 = iio[-WS(ios, 7)];
+		    T2 = W[6];
+		    T4 = W[7];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T16 = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T8, Ta, T7, T9;
+		    T8 = rio[WS(ios, 8)];
+		    Ta = iio[-WS(ios, 3)];
+		    T7 = W[14];
+		    T9 = W[15];
+		    Tb = FMA(T7, T8, T9 * Ta);
+		    T17 = FNMS(T9, T8, T7 * Ta);
+	       }
+	       T18 = KP866025403 * (T16 - T17);
+	       T22 = KP866025403 * (Tb - T6);
+	       Tc = T6 + Tb;
+	       T15 = FNMS(KP500000000, Tc, T1);
+	       T1V = T16 + T17;
+	       T23 = FNMS(KP500000000, T1V, T1W);
+	  }
+	  {
+	       E TK, T1i, TF, T1h;
+	       {
+		    E Tx, Tz, Tw, Ty;
+		    Tx = rio[WS(ios, 3)];
+		    Tz = iio[-WS(ios, 8)];
+		    Tw = W[4];
+		    Ty = W[5];
+		    TA = FMA(Tw, Tx, Ty * Tz);
+		    T1y = FNMS(Ty, Tx, Tw * Tz);
+	       }
+	       {
+		    E TH, TJ, TG, TI;
+		    TH = rio[WS(ios, 11)];
+		    TJ = iio[0];
+		    TG = W[20];
+		    TI = W[21];
+		    TK = FMA(TG, TH, TI * TJ);
+		    T1i = FNMS(TI, TH, TG * TJ);
+	       }
+	       {
+		    E TC, TE, TB, TD;
+		    TC = rio[WS(ios, 7)];
+		    TE = iio[-WS(ios, 4)];
+		    TB = W[12];
+		    TD = W[13];
+		    TF = FMA(TB, TC, TD * TE);
+		    T1h = FNMS(TD, TC, TB * TE);
+	       }
+	       T1j = KP866025403 * (T1h - T1i);
+	       T1B = KP866025403 * (TK - TF);
+	       TL = TF + TK;
+	       T1g = FNMS(KP500000000, TL, TA);
+	       T1z = T1h + T1i;
+	       T1A = FNMS(KP500000000, T1z, T1y);
+	  }
+	  {
+	       E Ts, T1c, Tn, T1b;
+	       {
+		    E Tf, Th, Te, Tg;
+		    Tf = rio[WS(ios, 6)];
+		    Th = iio[-WS(ios, 5)];
+		    Te = W[10];
+		    Tg = W[11];
+		    Ti = FMA(Te, Tf, Tg * Th);
+		    T1S = FNMS(Tg, Tf, Te * Th);
+	       }
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = rio[WS(ios, 2)];
+		    Tr = iio[-WS(ios, 9)];
+		    To = W[2];
+		    Tq = W[3];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    T1c = FNMS(Tq, Tp, To * Tr);
+	       }
+	       {
+		    E Tk, Tm, Tj, Tl;
+		    Tk = rio[WS(ios, 10)];
+		    Tm = iio[-WS(ios, 1)];
+		    Tj = W[18];
+		    Tl = W[19];
+		    Tn = FMA(Tj, Tk, Tl * Tm);
+		    T1b = FNMS(Tl, Tk, Tj * Tm);
+	       }
+	       T1d = KP866025403 * (T1b - T1c);
+	       T25 = KP866025403 * (Ts - Tn);
+	       Tt = Tn + Ts;
+	       T1a = FNMS(KP500000000, Tt, Ti);
+	       T1T = T1b + T1c;
+	       T26 = FNMS(KP500000000, T1T, T1S);
+	  }
+	  {
+	       E T11, T1n, TW, T1m;
+	       {
+		    E TO, TQ, TN, TP;
+		    TO = rio[WS(ios, 9)];
+		    TQ = iio[-WS(ios, 2)];
+		    TN = W[16];
+		    TP = W[17];
+		    TR = FMA(TN, TO, TP * TQ);
+		    T1E = FNMS(TP, TO, TN * TQ);
+	       }
+	       {
+		    E TY, T10, TX, TZ;
+		    TY = rio[WS(ios, 5)];
+		    T10 = iio[-WS(ios, 6)];
+		    TX = W[8];
+		    TZ = W[9];
+		    T11 = FMA(TX, TY, TZ * T10);
+		    T1n = FNMS(TZ, TY, TX * T10);
+	       }
+	       {
+		    E TT, TV, TS, TU;
+		    TT = rio[WS(ios, 1)];
+		    TV = iio[-WS(ios, 10)];
+		    TS = W[0];
+		    TU = W[1];
+		    TW = FMA(TS, TT, TU * TV);
+		    T1m = FNMS(TU, TT, TS * TV);
+	       }
+	       T1o = KP866025403 * (T1m - T1n);
+	       T1D = KP866025403 * (T11 - TW);
+	       T12 = TW + T11;
+	       T1l = FNMS(KP500000000, T12, TR);
+	       T1F = T1m + T1n;
+	       T1G = FNMS(KP500000000, T1F, T1E);
+	  }
+	  {
+	       E Tv, T1N, T1Y, T20, T14, T1Z, T1Q, T1R;
+	       {
+		    E Td, Tu, T1U, T1X;
+		    Td = T1 + Tc;
+		    Tu = Ti + Tt;
+		    Tv = Td + Tu;
+		    T1N = Td - Tu;
+		    T1U = T1S + T1T;
+		    T1X = T1V + T1W;
+		    T1Y = T1U + T1X;
+		    T20 = T1X - T1U;
+	       }
+	       {
+		    E TM, T13, T1O, T1P;
+		    TM = TA + TL;
+		    T13 = TR + T12;
+		    T14 = TM + T13;
+		    T1Z = TM - T13;
+		    T1O = T1y + T1z;
+		    T1P = T1E + T1F;
+		    T1Q = T1O - T1P;
+		    T1R = T1O + T1P;
+	       }
+	       iio[-WS(ios, 6)] = Tv - T14;
+	       rio[WS(ios, 6)] = T1R - T1Y;
+	       rio[0] = Tv + T14;
+	       iio[0] = T1R + T1Y;
+	       rio[WS(ios, 3)] = T1N - T1Q;
+	       iio[-WS(ios, 3)] = T1Z + T20;
+	       iio[-WS(ios, 9)] = T1N + T1Q;
+	       rio[WS(ios, 9)] = T1Z - T20;
+	  }
+	  {
+	       E T1t, T1J, T28, T2a, T1w, T21, T1M, T29;
+	       {
+		    E T1r, T1s, T24, T27;
+		    T1r = T15 + T18;
+		    T1s = T1a + T1d;
+		    T1t = T1r + T1s;
+		    T1J = T1r - T1s;
+		    T24 = T22 + T23;
+		    T27 = T25 + T26;
+		    T28 = T24 - T27;
+		    T2a = T27 + T24;
+	       }
+	       {
+		    E T1u, T1v, T1K, T1L;
+		    T1u = T1g + T1j;
+		    T1v = T1l + T1o;
+		    T1w = T1u + T1v;
+		    T21 = T1v - T1u;
+		    T1K = T1B + T1A;
+		    T1L = T1D + T1G;
+		    T1M = T1K - T1L;
+		    T29 = T1K + T1L;
+	       }
+	       iio[-WS(ios, 10)] = T1t - T1w;
+	       rio[WS(ios, 10)] = T29 - T2a;
+	       rio[WS(ios, 4)] = T1t + T1w;
+	       iio[-WS(ios, 4)] = T29 + T2a;
+	       iio[-WS(ios, 7)] = T1J - T1M;
+	       rio[WS(ios, 7)] = T21 - T28;
+	       rio[WS(ios, 1)] = T1J + T1M;
+	       iio[-WS(ios, 1)] = T21 + T28;
+	  }
+	  {
+	       E T1f, T1x, T2e, T2g, T1q, T2f, T1I, T2b;
+	       {
+		    E T19, T1e, T2c, T2d;
+		    T19 = T15 - T18;
+		    T1e = T1a - T1d;
+		    T1f = T19 + T1e;
+		    T1x = T19 - T1e;
+		    T2c = T26 - T25;
+		    T2d = T23 - T22;
+		    T2e = T2c + T2d;
+		    T2g = T2d - T2c;
+	       }
+	       {
+		    E T1k, T1p, T1C, T1H;
+		    T1k = T1g - T1j;
+		    T1p = T1l - T1o;
+		    T1q = T1k + T1p;
+		    T2f = T1p - T1k;
+		    T1C = T1A - T1B;
+		    T1H = T1D - T1G;
+		    T1I = T1C + T1H;
+		    T2b = T1H - T1C;
+	       }
+	       rio[WS(ios, 2)] = T1f - T1q;
+	       iio[-WS(ios, 2)] = T2b + T2e;
+	       iio[-WS(ios, 8)] = T1f + T1q;
+	       rio[WS(ios, 8)] = T2b - T2e;
+	       iio[-WS(ios, 11)] = T1x - T1I;
+	       rio[WS(ios, 11)] = T2f - T2g;
+	       rio[WS(ios, 5)] = T1x + T1I;
+	       iio[-WS(ios, 5)] = T2f + T2g;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 12},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 12, "hf_12", twinstr, {88, 30, 30, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_12) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_12, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_15.c b/src/fftw3/rdft/codelets/r2hc/hf_15.c
new file mode 100644
index 0000000..71a3b40
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_15.c
@@ -0,0 +1,405 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:07 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 15 -dit -name hf_15 -include hf.h */
+
+/*
+ * This function contains 184 FP additions, 112 FP multiplications,
+ * (or, 128 additions, 56 multiplications, 56 fused multiply/add),
+ * 65 stack variables, and 60 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_15.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_15.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_15.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_15(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 28) {
+	  E T1q, T34, Td, T1n, T2S, T35, T13, T1k, T1l, T2E, T2F, T2O, T1H, T1T, T2k;
+	  E T2t, T2f, T2s, T1M, T1U, Tu, TL, TM, T2H, T2I, T2N, T1w, T1Q, T29, T2w;
+	  E T24, T2v, T1B, T1R;
+	  {
+	       E T1, T2R, T6, T1o, Tb, T1p, Tc, T2Q;
+	       T1 = rio[0];
+	       T2R = iio[-WS(ios, 14)];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = rio[WS(ios, 5)];
+		    T5 = iio[-WS(ios, 9)];
+		    T2 = W[8];
+		    T4 = W[9];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T1o = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T8, Ta, T7, T9;
+		    T8 = rio[WS(ios, 10)];
+		    Ta = iio[-WS(ios, 4)];
+		    T7 = W[18];
+		    T9 = W[19];
+		    Tb = FMA(T7, T8, T9 * Ta);
+		    T1p = FNMS(T9, T8, T7 * Ta);
+	       }
+	       T1q = KP866025403 * (T1o - T1p);
+	       T34 = KP866025403 * (Tb - T6);
+	       Tc = T6 + Tb;
+	       Td = T1 + Tc;
+	       T1n = FNMS(KP500000000, Tc, T1);
+	       T2Q = T1o + T1p;
+	       T2S = T2Q + T2R;
+	       T35 = FNMS(KP500000000, T2Q, T2R);
+	  }
+	  {
+	       E TR, T2c, T18, T2h, TW, T1E, T11, T1F, T12, T2d, T1d, T1J, T1i, T1K, T1j;
+	       E T2i;
+	       {
+		    E TO, TQ, TN, TP;
+		    TO = rio[WS(ios, 6)];
+		    TQ = iio[-WS(ios, 8)];
+		    TN = W[10];
+		    TP = W[11];
+		    TR = FMA(TN, TO, TP * TQ);
+		    T2c = FNMS(TP, TO, TN * TQ);
+	       }
+	       {
+		    E T15, T17, T14, T16;
+		    T15 = rio[WS(ios, 9)];
+		    T17 = iio[-WS(ios, 5)];
+		    T14 = W[16];
+		    T16 = W[17];
+		    T18 = FMA(T14, T15, T16 * T17);
+		    T2h = FNMS(T16, T15, T14 * T17);
+	       }
+	       {
+		    E TT, TV, TS, TU;
+		    TT = rio[WS(ios, 11)];
+		    TV = iio[-WS(ios, 3)];
+		    TS = W[20];
+		    TU = W[21];
+		    TW = FMA(TS, TT, TU * TV);
+		    T1E = FNMS(TU, TT, TS * TV);
+	       }
+	       {
+		    E TY, T10, TX, TZ;
+		    TY = rio[WS(ios, 1)];
+		    T10 = iio[-WS(ios, 13)];
+		    TX = W[0];
+		    TZ = W[1];
+		    T11 = FMA(TX, TY, TZ * T10);
+		    T1F = FNMS(TZ, TY, TX * T10);
+	       }
+	       T12 = TW + T11;
+	       T2d = T1E + T1F;
+	       {
+		    E T1a, T1c, T19, T1b;
+		    T1a = rio[WS(ios, 14)];
+		    T1c = iio[0];
+		    T19 = W[26];
+		    T1b = W[27];
+		    T1d = FMA(T19, T1a, T1b * T1c);
+		    T1J = FNMS(T1b, T1a, T19 * T1c);
+	       }
+	       {
+		    E T1f, T1h, T1e, T1g;
+		    T1f = rio[WS(ios, 4)];
+		    T1h = iio[-WS(ios, 10)];
+		    T1e = W[6];
+		    T1g = W[7];
+		    T1i = FMA(T1e, T1f, T1g * T1h);
+		    T1K = FNMS(T1g, T1f, T1e * T1h);
+	       }
+	       T1j = T1d + T1i;
+	       T2i = T1J + T1K;
+	       {
+		    E T1D, T1G, T2g, T2j;
+		    T13 = TR + T12;
+		    T1k = T18 + T1j;
+		    T1l = T13 + T1k;
+		    T2E = T2c + T2d;
+		    T2F = T2h + T2i;
+		    T2O = T2E + T2F;
+		    T1D = FNMS(KP500000000, T12, TR);
+		    T1G = KP866025403 * (T1E - T1F);
+		    T1H = T1D - T1G;
+		    T1T = T1D + T1G;
+		    T2g = KP866025403 * (T1d - T1i);
+		    T2j = FNMS(KP500000000, T2i, T2h);
+		    T2k = T2g - T2j;
+		    T2t = T2g + T2j;
+		    {
+			 E T2b, T2e, T1I, T1L;
+			 T2b = KP866025403 * (T11 - TW);
+			 T2e = FNMS(KP500000000, T2d, T2c);
+			 T2f = T2b + T2e;
+			 T2s = T2e - T2b;
+			 T1I = FNMS(KP500000000, T1j, T18);
+			 T1L = KP866025403 * (T1J - T1K);
+			 T1M = T1I - T1L;
+			 T1U = T1I + T1L;
+		    }
+	       }
+	  }
+	  {
+	       E Ti, T21, Tz, T26, TE, T1y, TJ, T1z, TK, T27, Tn, T1t, Ts, T1u, Tt;
+	       E T22;
+	       {
+		    E Tf, Th, Te, Tg;
+		    Tf = rio[WS(ios, 3)];
+		    Th = iio[-WS(ios, 11)];
+		    Te = W[4];
+		    Tg = W[5];
+		    Ti = FMA(Te, Tf, Tg * Th);
+		    T21 = FNMS(Tg, Tf, Te * Th);
+	       }
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = rio[WS(ios, 12)];
+		    Ty = iio[-WS(ios, 2)];
+		    Tv = W[22];
+		    Tx = W[23];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T26 = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = rio[WS(ios, 2)];
+		    TD = iio[-WS(ios, 12)];
+		    TA = W[2];
+		    TC = W[3];
+		    TE = FMA(TA, TB, TC * TD);
+		    T1y = FNMS(TC, TB, TA * TD);
+	       }
+	       {
+		    E TG, TI, TF, TH;
+		    TG = rio[WS(ios, 7)];
+		    TI = iio[-WS(ios, 7)];
+		    TF = W[12];
+		    TH = W[13];
+		    TJ = FMA(TF, TG, TH * TI);
+		    T1z = FNMS(TH, TG, TF * TI);
+	       }
+	       TK = TE + TJ;
+	       T27 = T1y + T1z;
+	       {
+		    E Tk, Tm, Tj, Tl;
+		    Tk = rio[WS(ios, 8)];
+		    Tm = iio[-WS(ios, 6)];
+		    Tj = W[14];
+		    Tl = W[15];
+		    Tn = FMA(Tj, Tk, Tl * Tm);
+		    T1t = FNMS(Tl, Tk, Tj * Tm);
+	       }
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = rio[WS(ios, 13)];
+		    Tr = iio[-WS(ios, 1)];
+		    To = W[24];
+		    Tq = W[25];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    T1u = FNMS(Tq, Tp, To * Tr);
+	       }
+	       Tt = Tn + Ts;
+	       T22 = T1t + T1u;
+	       {
+		    E T1s, T1v, T25, T28;
+		    Tu = Ti + Tt;
+		    TL = Tz + TK;
+		    TM = Tu + TL;
+		    T2H = T21 + T22;
+		    T2I = T26 + T27;
+		    T2N = T2H + T2I;
+		    T1s = FNMS(KP500000000, Tt, Ti);
+		    T1v = KP866025403 * (T1t - T1u);
+		    T1w = T1s - T1v;
+		    T1Q = T1s + T1v;
+		    T25 = KP866025403 * (TJ - TE);
+		    T28 = FNMS(KP500000000, T27, T26);
+		    T29 = T25 + T28;
+		    T2w = T28 - T25;
+		    {
+			 E T20, T23, T1x, T1A;
+			 T20 = KP866025403 * (Ts - Tn);
+			 T23 = FNMS(KP500000000, T22, T21);
+			 T24 = T20 + T23;
+			 T2v = T23 - T20;
+			 T1x = FNMS(KP500000000, TK, Tz);
+			 T1A = KP866025403 * (T1y - T1z);
+			 T1B = T1x - T1A;
+			 T1R = T1x + T1A;
+		    }
+	       }
+	  }
+	  {
+	       E T2C, T1m, T2B, T2K, T2M, T2G, T2J, T2L, T2D;
+	       T2C = KP559016994 * (TM - T1l);
+	       T1m = TM + T1l;
+	       T2B = FNMS(KP250000000, T1m, Td);
+	       T2G = T2E - T2F;
+	       T2J = T2H - T2I;
+	       T2K = FNMS(KP587785252, T2J, KP951056516 * T2G);
+	       T2M = FMA(KP951056516, T2J, KP587785252 * T2G);
+	       rio[0] = Td + T1m;
+	       T2L = T2C + T2B;
+	       iio[-WS(ios, 9)] = T2L - T2M;
+	       rio[WS(ios, 6)] = T2L + T2M;
+	       T2D = T2B - T2C;
+	       iio[-WS(ios, 12)] = T2D - T2K;
+	       rio[WS(ios, 3)] = T2D + T2K;
+	  }
+	  {
+	       E T2X, T2P, T2W, T2V, T2Z, T2T, T2U, T30, T2Y;
+	       T2X = KP559016994 * (T2N - T2O);
+	       T2P = T2N + T2O;
+	       T2W = FNMS(KP250000000, T2P, T2S);
+	       T2T = Tu - TL;
+	       T2U = T1k - T13;
+	       T2V = FMA(KP587785252, T2T, KP951056516 * T2U);
+	       T2Z = FNMS(KP951056516, T2T, KP587785252 * T2U);
+	       iio[0] = T2P + T2S;
+	       T30 = T2X + T2W;
+	       rio[WS(ios, 9)] = T2Z - T30;
+	       iio[-WS(ios, 6)] = T2Z + T30;
+	       T2Y = T2W - T2X;
+	       rio[WS(ios, 12)] = T2V - T2Y;
+	       iio[-WS(ios, 3)] = T2V + T2Y;
+	  }
+	  {
+	       E T2y, T2A, T1r, T1O, T2p, T2q, T2z, T2r;
+	       {
+		    E T2u, T2x, T1C, T1N;
+		    T2u = T2s - T2t;
+		    T2x = T2v - T2w;
+		    T2y = FNMS(KP587785252, T2x, KP951056516 * T2u);
+		    T2A = FMA(KP951056516, T2x, KP587785252 * T2u);
+		    T1r = T1n - T1q;
+		    T1C = T1w + T1B;
+		    T1N = T1H + T1M;
+		    T1O = T1C + T1N;
+		    T2p = FNMS(KP250000000, T1O, T1r);
+		    T2q = KP559016994 * (T1C - T1N);
+	       }
+	       rio[WS(ios, 5)] = T1r + T1O;
+	       T2z = T2q + T2p;
+	       iio[-WS(ios, 14)] = T2z - T2A;
+	       iio[-WS(ios, 11)] = T2z + T2A;
+	       T2r = T2p - T2q;
+	       rio[WS(ios, 2)] = T2r - T2y;
+	       iio[-WS(ios, 8)] = T2r + T2y;
+	  }
+	  {
+	       E T3h, T3p, T3l, T3m, T3k, T3n, T3q, T3o;
+	       {
+		    E T3f, T3g, T3i, T3j;
+		    T3f = T1w - T1B;
+		    T3g = T1H - T1M;
+		    T3h = FMA(KP951056516, T3f, KP587785252 * T3g);
+		    T3p = FNMS(KP587785252, T3f, KP951056516 * T3g);
+		    T3l = T35 - T34;
+		    T3i = T2s + T2t;
+		    T3j = T2v + T2w;
+		    T3m = T3j + T3i;
+		    T3k = KP559016994 * (T3i - T3j);
+		    T3n = FNMS(KP250000000, T3m, T3l);
+	       }
+	       iio[-WS(ios, 5)] = T3m + T3l;
+	       T3q = T3k + T3n;
+	       rio[WS(ios, 8)] = T3p - T3q;
+	       iio[-WS(ios, 2)] = T3p + T3q;
+	       T3o = T3k - T3n;
+	       rio[WS(ios, 11)] = T3h + T3o;
+	       rio[WS(ios, 14)] = T3o - T3h;
+	  }
+	  {
+	       E T3c, T3d, T36, T33, T37, T38, T3e, T39;
+	       {
+		    E T3a, T3b, T31, T32;
+		    T3a = T1Q - T1R;
+		    T3b = T1T - T1U;
+		    T3c = FMA(KP951056516, T3a, KP587785252 * T3b);
+		    T3d = FNMS(KP587785252, T3a, KP951056516 * T3b);
+		    T36 = T34 + T35;
+		    T31 = T2k - T2f;
+		    T32 = T24 + T29;
+		    T33 = T31 - T32;
+		    T37 = KP559016994 * (T32 + T31);
+		    T38 = FMA(KP250000000, T33, T36);
+	       }
+	       rio[WS(ios, 10)] = T33 - T36;
+	       T3e = T38 - T37;
+	       rio[WS(ios, 13)] = T3d - T3e;
+	       iio[-WS(ios, 7)] = T3d + T3e;
+	       T39 = T37 + T38;
+	       iio[-WS(ios, 1)] = T39 - T3c;
+	       iio[-WS(ios, 4)] = T3c + T39;
+	  }
+	  {
+	       E T2m, T2o, T1P, T1W, T1X, T1Y, T2n, T1Z;
+	       {
+		    E T2a, T2l, T1S, T1V;
+		    T2a = T24 - T29;
+		    T2l = T2f + T2k;
+		    T2m = FMA(KP951056516, T2a, KP587785252 * T2l);
+		    T2o = FNMS(KP587785252, T2a, KP951056516 * T2l);
+		    T1P = T1n + T1q;
+		    T1S = T1Q + T1R;
+		    T1V = T1T + T1U;
+		    T1W = T1S + T1V;
+		    T1X = KP559016994 * (T1S - T1V);
+		    T1Y = FNMS(KP250000000, T1W, T1P);
+	       }
+	       iio[-WS(ios, 10)] = T1P + T1W;
+	       T2n = T1Y - T1X;
+	       rio[WS(ios, 7)] = T2n - T2o;
+	       iio[-WS(ios, 13)] = T2n + T2o;
+	       T1Z = T1X + T1Y;
+	       rio[WS(ios, 4)] = T1Z - T2m;
+	       rio[WS(ios, 1)] = T1Z + T2m;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 15},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 15, "hf_15", twinstr, {128, 56, 56, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_15) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_15, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_16.c b/src/fftw3/rdft/codelets/r2hc/hf_16.c
new file mode 100644
index 0000000..ff3145d
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_16.c
@@ -0,0 +1,406 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:07 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 16 -dit -name hf_16 -include hf.h */
+
+/*
+ * This function contains 174 FP additions, 84 FP multiplications,
+ * (or, 136 additions, 46 multiplications, 38 fused multiply/add),
+ * 52 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_16(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 30) {
+	  E T7, T37, T1t, T2U, Ti, T38, T1w, T2R, Tu, T2s, T1C, T2c, TF, T2t, T1H;
+	  E T2d, TS, T13, T2w, T2x, T2y, T2z, T1O, T2g, T1T, T2h, T1f, T1q, T2B, T2C;
+	  E T2D, T2E, T1Z, T2j, T24, T2k;
+	  {
+	       E T1, T2T, T6, T2S;
+	       T1 = rio[0];
+	       T2T = iio[-WS(ios, 15)];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = rio[WS(ios, 8)];
+		    T5 = iio[-WS(ios, 7)];
+		    T2 = W[14];
+		    T4 = W[15];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T2S = FNMS(T4, T3, T2 * T5);
+	       }
+	       T7 = T1 + T6;
+	       T37 = T2T - T2S;
+	       T1t = T1 - T6;
+	       T2U = T2S + T2T;
+	  }
+	  {
+	       E Tc, T1u, Th, T1v;
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = rio[WS(ios, 4)];
+		    Tb = iio[-WS(ios, 11)];
+		    T8 = W[6];
+		    Ta = W[7];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    T1u = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = rio[WS(ios, 12)];
+		    Tg = iio[-WS(ios, 3)];
+		    Td = W[22];
+		    Tf = W[23];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    T1v = FNMS(Tf, Te, Td * Tg);
+	       }
+	       Ti = Tc + Th;
+	       T38 = Tc - Th;
+	       T1w = T1u - T1v;
+	       T2R = T1u + T1v;
+	  }
+	  {
+	       E To, T1y, Tt, T1z, T1A, T1B;
+	       {
+		    E Tl, Tn, Tk, Tm;
+		    Tl = rio[WS(ios, 2)];
+		    Tn = iio[-WS(ios, 13)];
+		    Tk = W[2];
+		    Tm = W[3];
+		    To = FMA(Tk, Tl, Tm * Tn);
+		    T1y = FNMS(Tm, Tl, Tk * Tn);
+	       }
+	       {
+		    E Tq, Ts, Tp, Tr;
+		    Tq = rio[WS(ios, 10)];
+		    Ts = iio[-WS(ios, 5)];
+		    Tp = W[18];
+		    Tr = W[19];
+		    Tt = FMA(Tp, Tq, Tr * Ts);
+		    T1z = FNMS(Tr, Tq, Tp * Ts);
+	       }
+	       Tu = To + Tt;
+	       T2s = T1y + T1z;
+	       T1A = T1y - T1z;
+	       T1B = To - Tt;
+	       T1C = T1A - T1B;
+	       T2c = T1B + T1A;
+	  }
+	  {
+	       E Tz, T1E, TE, T1F, T1D, T1G;
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = rio[WS(ios, 14)];
+		    Ty = iio[-WS(ios, 1)];
+		    Tv = W[26];
+		    Tx = W[27];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T1E = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = rio[WS(ios, 6)];
+		    TD = iio[-WS(ios, 9)];
+		    TA = W[10];
+		    TC = W[11];
+		    TE = FMA(TA, TB, TC * TD);
+		    T1F = FNMS(TC, TB, TA * TD);
+	       }
+	       TF = Tz + TE;
+	       T2t = T1E + T1F;
+	       T1D = Tz - TE;
+	       T1G = T1E - T1F;
+	       T1H = T1D + T1G;
+	       T2d = T1D - T1G;
+	  }
+	  {
+	       E TM, T1K, T12, T1R, TR, T1L, TX, T1Q;
+	       {
+		    E TJ, TL, TI, TK;
+		    TJ = rio[WS(ios, 1)];
+		    TL = iio[-WS(ios, 14)];
+		    TI = W[0];
+		    TK = W[1];
+		    TM = FMA(TI, TJ, TK * TL);
+		    T1K = FNMS(TK, TJ, TI * TL);
+	       }
+	       {
+		    E TZ, T11, TY, T10;
+		    TZ = rio[WS(ios, 13)];
+		    T11 = iio[-WS(ios, 2)];
+		    TY = W[24];
+		    T10 = W[25];
+		    T12 = FMA(TY, TZ, T10 * T11);
+		    T1R = FNMS(T10, TZ, TY * T11);
+	       }
+	       {
+		    E TO, TQ, TN, TP;
+		    TO = rio[WS(ios, 9)];
+		    TQ = iio[-WS(ios, 6)];
+		    TN = W[16];
+		    TP = W[17];
+		    TR = FMA(TN, TO, TP * TQ);
+		    T1L = FNMS(TP, TO, TN * TQ);
+	       }
+	       {
+		    E TU, TW, TT, TV;
+		    TU = rio[WS(ios, 5)];
+		    TW = iio[-WS(ios, 10)];
+		    TT = W[8];
+		    TV = W[9];
+		    TX = FMA(TT, TU, TV * TW);
+		    T1Q = FNMS(TV, TU, TT * TW);
+	       }
+	       TS = TM + TR;
+	       T13 = TX + T12;
+	       T2w = TS - T13;
+	       T2x = T1K + T1L;
+	       T2y = T1Q + T1R;
+	       T2z = T2x - T2y;
+	       {
+		    E T1M, T1N, T1P, T1S;
+		    T1M = T1K - T1L;
+		    T1N = TX - T12;
+		    T1O = T1M + T1N;
+		    T2g = T1M - T1N;
+		    T1P = TM - TR;
+		    T1S = T1Q - T1R;
+		    T1T = T1P - T1S;
+		    T2h = T1P + T1S;
+	       }
+	  }
+	  {
+	       E T19, T20, T1p, T1X, T1e, T21, T1k, T1W;
+	       {
+		    E T16, T18, T15, T17;
+		    T16 = rio[WS(ios, 15)];
+		    T18 = iio[0];
+		    T15 = W[28];
+		    T17 = W[29];
+		    T19 = FMA(T15, T16, T17 * T18);
+		    T20 = FNMS(T17, T16, T15 * T18);
+	       }
+	       {
+		    E T1m, T1o, T1l, T1n;
+		    T1m = rio[WS(ios, 11)];
+		    T1o = iio[-WS(ios, 4)];
+		    T1l = W[20];
+		    T1n = W[21];
+		    T1p = FMA(T1l, T1m, T1n * T1o);
+		    T1X = FNMS(T1n, T1m, T1l * T1o);
+	       }
+	       {
+		    E T1b, T1d, T1a, T1c;
+		    T1b = rio[WS(ios, 7)];
+		    T1d = iio[-WS(ios, 8)];
+		    T1a = W[12];
+		    T1c = W[13];
+		    T1e = FMA(T1a, T1b, T1c * T1d);
+		    T21 = FNMS(T1c, T1b, T1a * T1d);
+	       }
+	       {
+		    E T1h, T1j, T1g, T1i;
+		    T1h = rio[WS(ios, 3)];
+		    T1j = iio[-WS(ios, 12)];
+		    T1g = W[4];
+		    T1i = W[5];
+		    T1k = FMA(T1g, T1h, T1i * T1j);
+		    T1W = FNMS(T1i, T1h, T1g * T1j);
+	       }
+	       T1f = T19 + T1e;
+	       T1q = T1k + T1p;
+	       T2B = T1f - T1q;
+	       T2C = T20 + T21;
+	       T2D = T1W + T1X;
+	       T2E = T2C - T2D;
+	       {
+		    E T1V, T1Y, T22, T23;
+		    T1V = T19 - T1e;
+		    T1Y = T1W - T1X;
+		    T1Z = T1V - T1Y;
+		    T2j = T1V + T1Y;
+		    T22 = T20 - T21;
+		    T23 = T1k - T1p;
+		    T24 = T22 + T23;
+		    T2k = T22 - T23;
+	       }
+	  }
+	  {
+	       E T1J, T27, T3g, T3i, T26, T3h, T2a, T3d;
+	       {
+		    E T1x, T1I, T3e, T3f;
+		    T1x = T1t - T1w;
+		    T1I = KP707106781 * (T1C - T1H);
+		    T1J = T1x + T1I;
+		    T27 = T1x - T1I;
+		    T3e = KP707106781 * (T2d - T2c);
+		    T3f = T38 + T37;
+		    T3g = T3e + T3f;
+		    T3i = T3f - T3e;
+	       }
+	       {
+		    E T1U, T25, T28, T29;
+		    T1U = FMA(KP923879532, T1O, KP382683432 * T1T);
+		    T25 = FNMS(KP923879532, T24, KP382683432 * T1Z);
+		    T26 = T1U + T25;
+		    T3h = T25 - T1U;
+		    T28 = FNMS(KP923879532, T1T, KP382683432 * T1O);
+		    T29 = FMA(KP382683432, T24, KP923879532 * T1Z);
+		    T2a = T28 - T29;
+		    T3d = T28 + T29;
+	       }
+	       iio[-WS(ios, 11)] = T1J - T26;
+	       rio[WS(ios, 11)] = T3d - T3g;
+	       rio[WS(ios, 3)] = T1J + T26;
+	       iio[-WS(ios, 3)] = T3d + T3g;
+	       iio[-WS(ios, 15)] = T27 - T2a;
+	       rio[WS(ios, 15)] = T3h - T3i;
+	       rio[WS(ios, 7)] = T27 + T2a;
+	       iio[-WS(ios, 7)] = T3h + T3i;
+	  }
+	  {
+	       E T2v, T2H, T32, T34, T2G, T33, T2K, T2Z;
+	       {
+		    E T2r, T2u, T30, T31;
+		    T2r = T7 - Ti;
+		    T2u = T2s - T2t;
+		    T2v = T2r + T2u;
+		    T2H = T2r - T2u;
+		    T30 = TF - Tu;
+		    T31 = T2U - T2R;
+		    T32 = T30 + T31;
+		    T34 = T31 - T30;
+	       }
+	       {
+		    E T2A, T2F, T2I, T2J;
+		    T2A = T2w + T2z;
+		    T2F = T2B - T2E;
+		    T2G = KP707106781 * (T2A + T2F);
+		    T33 = KP707106781 * (T2F - T2A);
+		    T2I = T2z - T2w;
+		    T2J = T2B + T2E;
+		    T2K = KP707106781 * (T2I - T2J);
+		    T2Z = KP707106781 * (T2I + T2J);
+	       }
+	       iio[-WS(ios, 10)] = T2v - T2G;
+	       rio[WS(ios, 10)] = T2Z - T32;
+	       rio[WS(ios, 2)] = T2v + T2G;
+	       iio[-WS(ios, 2)] = T2Z + T32;
+	       iio[-WS(ios, 14)] = T2H - T2K;
+	       rio[WS(ios, 14)] = T33 - T34;
+	       rio[WS(ios, 6)] = T2H + T2K;
+	       iio[-WS(ios, 6)] = T33 + T34;
+	  }
+	  {
+	       E T2f, T2n, T3a, T3c, T2m, T3b, T2q, T35;
+	       {
+		    E T2b, T2e, T36, T39;
+		    T2b = T1t + T1w;
+		    T2e = KP707106781 * (T2c + T2d);
+		    T2f = T2b + T2e;
+		    T2n = T2b - T2e;
+		    T36 = KP707106781 * (T1C + T1H);
+		    T39 = T37 - T38;
+		    T3a = T36 + T39;
+		    T3c = T39 - T36;
+	       }
+	       {
+		    E T2i, T2l, T2o, T2p;
+		    T2i = FMA(KP382683432, T2g, KP923879532 * T2h);
+		    T2l = FNMS(KP382683432, T2k, KP923879532 * T2j);
+		    T2m = T2i + T2l;
+		    T3b = T2l - T2i;
+		    T2o = FNMS(KP382683432, T2h, KP923879532 * T2g);
+		    T2p = FMA(KP923879532, T2k, KP382683432 * T2j);
+		    T2q = T2o - T2p;
+		    T35 = T2o + T2p;
+	       }
+	       iio[-WS(ios, 9)] = T2f - T2m;
+	       rio[WS(ios, 9)] = T35 - T3a;
+	       rio[WS(ios, 1)] = T2f + T2m;
+	       iio[-WS(ios, 1)] = T35 + T3a;
+	       iio[-WS(ios, 13)] = T2n - T2q;
+	       rio[WS(ios, 13)] = T3b - T3c;
+	       rio[WS(ios, 5)] = T2n + T2q;
+	       iio[-WS(ios, 5)] = T3b + T3c;
+	  }
+	  {
+	       E TH, T2L, T2W, T2Y, T1s, T2X, T2O, T2P;
+	       {
+		    E Tj, TG, T2Q, T2V;
+		    Tj = T7 + Ti;
+		    TG = Tu + TF;
+		    TH = Tj + TG;
+		    T2L = Tj - TG;
+		    T2Q = T2s + T2t;
+		    T2V = T2R + T2U;
+		    T2W = T2Q + T2V;
+		    T2Y = T2V - T2Q;
+	       }
+	       {
+		    E T14, T1r, T2M, T2N;
+		    T14 = TS + T13;
+		    T1r = T1f + T1q;
+		    T1s = T14 + T1r;
+		    T2X = T1r - T14;
+		    T2M = T2x + T2y;
+		    T2N = T2C + T2D;
+		    T2O = T2M - T2N;
+		    T2P = T2M + T2N;
+	       }
+	       iio[-WS(ios, 8)] = TH - T1s;
+	       rio[WS(ios, 8)] = T2P - T2W;
+	       rio[0] = TH + T1s;
+	       iio[0] = T2P + T2W;
+	       iio[-WS(ios, 12)] = T2L - T2O;
+	       rio[WS(ios, 12)] = T2X - T2Y;
+	       rio[WS(ios, 4)] = T2L + T2O;
+	       iio[-WS(ios, 4)] = T2X + T2Y;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 16},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 16, "hf_16", twinstr, {136, 46, 38, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_16) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_16, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_2.c b/src/fftw3/rdft/codelets/r2hc/hf_2.c
new file mode 100644
index 0000000..7de094c
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_2.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:51 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 2 -dit -name hf_2 -include hf.h */
+
+/*
+ * This function contains 6 FP additions, 4 FP multiplications,
+ * (or, 4 additions, 2 multiplications, 2 fused multiply/add),
+ * 9 stack variables, and 8 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_2.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_2.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_2.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_2(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 2) {
+	  E T1, T8, T6, T7;
+	  T1 = rio[0];
+	  T8 = iio[-WS(ios, 1)];
+	  {
+	       E T3, T5, T2, T4;
+	       T3 = rio[WS(ios, 1)];
+	       T5 = iio[0];
+	       T2 = W[0];
+	       T4 = W[1];
+	       T6 = FMA(T2, T3, T4 * T5);
+	       T7 = FNMS(T4, T3, T2 * T5);
+	  }
+	  iio[-WS(ios, 1)] = T1 - T6;
+	  rio[WS(ios, 1)] = T7 - T8;
+	  rio[0] = T1 + T6;
+	  iio[0] = T7 + T8;
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 2},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 2, "hf_2", twinstr, {4, 2, 2, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_2) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_2, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_3.c b/src/fftw3/rdft/codelets/r2hc/hf_3.c
new file mode 100644
index 0000000..fce963f
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_3.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:51 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 3 -dit -name hf_3 -include hf.h */
+
+/*
+ * This function contains 16 FP additions, 12 FP multiplications,
+ * (or, 10 additions, 6 multiplications, 6 fused multiply/add),
+ * 15 stack variables, and 12 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_3.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_3.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_3.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_3(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 4) {
+	  E T1, Ti, T6, Te, Tb, Tf, Tc, Th;
+	  T1 = rio[0];
+	  Ti = iio[-WS(ios, 2)];
+	  {
+	       E T3, T5, T2, T4;
+	       T3 = rio[WS(ios, 1)];
+	       T5 = iio[-WS(ios, 1)];
+	       T2 = W[0];
+	       T4 = W[1];
+	       T6 = FMA(T2, T3, T4 * T5);
+	       Te = FNMS(T4, T3, T2 * T5);
+	  }
+	  {
+	       E T8, Ta, T7, T9;
+	       T8 = rio[WS(ios, 2)];
+	       Ta = iio[0];
+	       T7 = W[2];
+	       T9 = W[3];
+	       Tb = FMA(T7, T8, T9 * Ta);
+	       Tf = FNMS(T9, T8, T7 * Ta);
+	  }
+	  Tc = T6 + Tb;
+	  Th = Te + Tf;
+	  rio[0] = T1 + Tc;
+	  iio[0] = Th + Ti;
+	  {
+	       E Td, Tg, Tj, Tk;
+	       Td = FNMS(KP500000000, Tc, T1);
+	       Tg = KP866025403 * (Te - Tf);
+	       iio[-WS(ios, 2)] = Td - Tg;
+	       rio[WS(ios, 1)] = Td + Tg;
+	       Tj = KP866025403 * (Tb - T6);
+	       Tk = FNMS(KP500000000, Th, Ti);
+	       rio[WS(ios, 2)] = Tj - Tk;
+	       iio[-WS(ios, 1)] = Tj + Tk;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 3},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 3, "hf_3", twinstr, {10, 6, 6, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_3) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_3, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_32.c b/src/fftw3/rdft/codelets/r2hc/hf_32.c
new file mode 100644
index 0000000..262ef43
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_32.c
@@ -0,0 +1,892 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:08 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 32 -dit -name hf_32 -include hf.h */
+
+/*
+ * This function contains 434 FP additions, 208 FP multiplications,
+ * (or, 340 additions, 114 multiplications, 94 fused multiply/add),
+ * 96 stack variables, and 128 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_32(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 62) {
+	  E Tj, T5F, T7C, T7Q, T35, T4T, T78, T7m, T2B, T67, T6e, T6O, T4b, T5d, T4s;
+	  E T5g, TG, T7l, T5I, T73, T3a, T4U, T3f, T4V, T1Q, T61, T5Y, T6J, T3K, T59;
+	  E T41, T56, T1r, T5P, T5S, T6F, T3x, T51, T3C, T52, T14, T5N, T5M, T6E, T3m;
+	  E T4Y, T3r, T4Z, T2d, T5Z, T64, T6K, T3V, T57, T44, T5a, T2Y, T6f, T6a, T6P;
+	  E T4m, T5h, T4v, T5e;
+	  {
+	       E T1, T76, T6, T75, Tc, T32, Th, T33;
+	       T1 = rio[0];
+	       T76 = iio[-WS(ios, 31)];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = rio[WS(ios, 16)];
+		    T5 = iio[-WS(ios, 15)];
+		    T2 = W[30];
+		    T4 = W[31];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T75 = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = rio[WS(ios, 8)];
+		    Tb = iio[-WS(ios, 23)];
+		    T8 = W[14];
+		    Ta = W[15];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    T32 = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = rio[WS(ios, 24)];
+		    Tg = iio[-WS(ios, 7)];
+		    Td = W[46];
+		    Tf = W[47];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    T33 = FNMS(Tf, Te, Td * Tg);
+	       }
+	       {
+		    E T7, Ti, T7A, T7B;
+		    T7 = T1 + T6;
+		    Ti = Tc + Th;
+		    Tj = T7 + Ti;
+		    T5F = T7 - Ti;
+		    T7A = T76 - T75;
+		    T7B = Tc - Th;
+		    T7C = T7A - T7B;
+		    T7Q = T7B + T7A;
+	       }
+	       {
+		    E T31, T34, T74, T77;
+		    T31 = T1 - T6;
+		    T34 = T32 - T33;
+		    T35 = T31 - T34;
+		    T4T = T31 + T34;
+		    T74 = T32 + T33;
+		    T77 = T75 + T76;
+		    T78 = T74 + T77;
+		    T7m = T77 - T74;
+	       }
+	  }
+	  {
+	       E T2j, T4o, T2z, T49, T2o, T4p, T2u, T48;
+	       {
+		    E T2g, T2i, T2f, T2h;
+		    T2g = rio[WS(ios, 31)];
+		    T2i = iio[0];
+		    T2f = W[60];
+		    T2h = W[61];
+		    T2j = FMA(T2f, T2g, T2h * T2i);
+		    T4o = FNMS(T2h, T2g, T2f * T2i);
+	       }
+	       {
+		    E T2w, T2y, T2v, T2x;
+		    T2w = rio[WS(ios, 23)];
+		    T2y = iio[-WS(ios, 8)];
+		    T2v = W[44];
+		    T2x = W[45];
+		    T2z = FMA(T2v, T2w, T2x * T2y);
+		    T49 = FNMS(T2x, T2w, T2v * T2y);
+	       }
+	       {
+		    E T2l, T2n, T2k, T2m;
+		    T2l = rio[WS(ios, 15)];
+		    T2n = iio[-WS(ios, 16)];
+		    T2k = W[28];
+		    T2m = W[29];
+		    T2o = FMA(T2k, T2l, T2m * T2n);
+		    T4p = FNMS(T2m, T2l, T2k * T2n);
+	       }
+	       {
+		    E T2r, T2t, T2q, T2s;
+		    T2r = rio[WS(ios, 7)];
+		    T2t = iio[-WS(ios, 24)];
+		    T2q = W[12];
+		    T2s = W[13];
+		    T2u = FMA(T2q, T2r, T2s * T2t);
+		    T48 = FNMS(T2s, T2r, T2q * T2t);
+	       }
+	       {
+		    E T2p, T2A, T6c, T6d;
+		    T2p = T2j + T2o;
+		    T2A = T2u + T2z;
+		    T2B = T2p + T2A;
+		    T67 = T2p - T2A;
+		    T6c = T4o + T4p;
+		    T6d = T48 + T49;
+		    T6e = T6c - T6d;
+		    T6O = T6c + T6d;
+	       }
+	       {
+		    E T47, T4a, T4q, T4r;
+		    T47 = T2j - T2o;
+		    T4a = T48 - T49;
+		    T4b = T47 - T4a;
+		    T5d = T47 + T4a;
+		    T4q = T4o - T4p;
+		    T4r = T2u - T2z;
+		    T4s = T4q + T4r;
+		    T5g = T4q - T4r;
+	       }
+	  }
+	  {
+	       E To, T36, TE, T3d, Tt, T37, Tz, T3c;
+	       {
+		    E Tl, Tn, Tk, Tm;
+		    Tl = rio[WS(ios, 4)];
+		    Tn = iio[-WS(ios, 27)];
+		    Tk = W[6];
+		    Tm = W[7];
+		    To = FMA(Tk, Tl, Tm * Tn);
+		    T36 = FNMS(Tm, Tl, Tk * Tn);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = rio[WS(ios, 12)];
+		    TD = iio[-WS(ios, 19)];
+		    TA = W[22];
+		    TC = W[23];
+		    TE = FMA(TA, TB, TC * TD);
+		    T3d = FNMS(TC, TB, TA * TD);
+	       }
+	       {
+		    E Tq, Ts, Tp, Tr;
+		    Tq = rio[WS(ios, 20)];
+		    Ts = iio[-WS(ios, 11)];
+		    Tp = W[38];
+		    Tr = W[39];
+		    Tt = FMA(Tp, Tq, Tr * Ts);
+		    T37 = FNMS(Tr, Tq, Tp * Ts);
+	       }
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = rio[WS(ios, 28)];
+		    Ty = iio[-WS(ios, 3)];
+		    Tv = W[54];
+		    Tx = W[55];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T3c = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E Tu, TF, T5G, T5H;
+		    Tu = To + Tt;
+		    TF = Tz + TE;
+		    TG = Tu + TF;
+		    T7l = TF - Tu;
+		    T5G = T36 + T37;
+		    T5H = T3c + T3d;
+		    T5I = T5G - T5H;
+		    T73 = T5G + T5H;
+	       }
+	       {
+		    E T38, T39, T3b, T3e;
+		    T38 = T36 - T37;
+		    T39 = To - Tt;
+		    T3a = T38 - T39;
+		    T4U = T39 + T38;
+		    T3b = Tz - TE;
+		    T3e = T3c - T3d;
+		    T3f = T3b + T3e;
+		    T4V = T3b - T3e;
+	       }
+	  }
+	  {
+	       E T1y, T3G, T1O, T3Z, T1D, T3H, T1J, T3Y;
+	       {
+		    E T1v, T1x, T1u, T1w;
+		    T1v = rio[WS(ios, 1)];
+		    T1x = iio[-WS(ios, 30)];
+		    T1u = W[0];
+		    T1w = W[1];
+		    T1y = FMA(T1u, T1v, T1w * T1x);
+		    T3G = FNMS(T1w, T1v, T1u * T1x);
+	       }
+	       {
+		    E T1L, T1N, T1K, T1M;
+		    T1L = rio[WS(ios, 25)];
+		    T1N = iio[-WS(ios, 6)];
+		    T1K = W[48];
+		    T1M = W[49];
+		    T1O = FMA(T1K, T1L, T1M * T1N);
+		    T3Z = FNMS(T1M, T1L, T1K * T1N);
+	       }
+	       {
+		    E T1A, T1C, T1z, T1B;
+		    T1A = rio[WS(ios, 17)];
+		    T1C = iio[-WS(ios, 14)];
+		    T1z = W[32];
+		    T1B = W[33];
+		    T1D = FMA(T1z, T1A, T1B * T1C);
+		    T3H = FNMS(T1B, T1A, T1z * T1C);
+	       }
+	       {
+		    E T1G, T1I, T1F, T1H;
+		    T1G = rio[WS(ios, 9)];
+		    T1I = iio[-WS(ios, 22)];
+		    T1F = W[16];
+		    T1H = W[17];
+		    T1J = FMA(T1F, T1G, T1H * T1I);
+		    T3Y = FNMS(T1H, T1G, T1F * T1I);
+	       }
+	       {
+		    E T1E, T1P, T5W, T5X;
+		    T1E = T1y + T1D;
+		    T1P = T1J + T1O;
+		    T1Q = T1E + T1P;
+		    T61 = T1E - T1P;
+		    T5W = T3G + T3H;
+		    T5X = T3Y + T3Z;
+		    T5Y = T5W - T5X;
+		    T6J = T5W + T5X;
+	       }
+	       {
+		    E T3I, T3J, T3X, T40;
+		    T3I = T3G - T3H;
+		    T3J = T1J - T1O;
+		    T3K = T3I + T3J;
+		    T59 = T3I - T3J;
+		    T3X = T1y - T1D;
+		    T40 = T3Y - T3Z;
+		    T41 = T3X - T40;
+		    T56 = T3X + T40;
+	       }
+	  }
+	  {
+	       E T19, T3t, T1p, T3A, T1e, T3u, T1k, T3z;
+	       {
+		    E T16, T18, T15, T17;
+		    T16 = rio[WS(ios, 30)];
+		    T18 = iio[-WS(ios, 1)];
+		    T15 = W[58];
+		    T17 = W[59];
+		    T19 = FMA(T15, T16, T17 * T18);
+		    T3t = FNMS(T17, T16, T15 * T18);
+	       }
+	       {
+		    E T1m, T1o, T1l, T1n;
+		    T1m = rio[WS(ios, 22)];
+		    T1o = iio[-WS(ios, 9)];
+		    T1l = W[42];
+		    T1n = W[43];
+		    T1p = FMA(T1l, T1m, T1n * T1o);
+		    T3A = FNMS(T1n, T1m, T1l * T1o);
+	       }
+	       {
+		    E T1b, T1d, T1a, T1c;
+		    T1b = rio[WS(ios, 14)];
+		    T1d = iio[-WS(ios, 17)];
+		    T1a = W[26];
+		    T1c = W[27];
+		    T1e = FMA(T1a, T1b, T1c * T1d);
+		    T3u = FNMS(T1c, T1b, T1a * T1d);
+	       }
+	       {
+		    E T1h, T1j, T1g, T1i;
+		    T1h = rio[WS(ios, 6)];
+		    T1j = iio[-WS(ios, 25)];
+		    T1g = W[10];
+		    T1i = W[11];
+		    T1k = FMA(T1g, T1h, T1i * T1j);
+		    T3z = FNMS(T1i, T1h, T1g * T1j);
+	       }
+	       {
+		    E T1f, T1q, T5Q, T5R;
+		    T1f = T19 + T1e;
+		    T1q = T1k + T1p;
+		    T1r = T1f + T1q;
+		    T5P = T1f - T1q;
+		    T5Q = T3t + T3u;
+		    T5R = T3z + T3A;
+		    T5S = T5Q - T5R;
+		    T6F = T5Q + T5R;
+	       }
+	       {
+		    E T3v, T3w, T3y, T3B;
+		    T3v = T3t - T3u;
+		    T3w = T1k - T1p;
+		    T3x = T3v + T3w;
+		    T51 = T3v - T3w;
+		    T3y = T19 - T1e;
+		    T3B = T3z - T3A;
+		    T3C = T3y - T3B;
+		    T52 = T3y + T3B;
+	       }
+	  }
+	  {
+	       E TM, T3i, T12, T3p, TR, T3j, TX, T3o;
+	       {
+		    E TJ, TL, TI, TK;
+		    TJ = rio[WS(ios, 2)];
+		    TL = iio[-WS(ios, 29)];
+		    TI = W[2];
+		    TK = W[3];
+		    TM = FMA(TI, TJ, TK * TL);
+		    T3i = FNMS(TK, TJ, TI * TL);
+	       }
+	       {
+		    E TZ, T11, TY, T10;
+		    TZ = rio[WS(ios, 26)];
+		    T11 = iio[-WS(ios, 5)];
+		    TY = W[50];
+		    T10 = W[51];
+		    T12 = FMA(TY, TZ, T10 * T11);
+		    T3p = FNMS(T10, TZ, TY * T11);
+	       }
+	       {
+		    E TO, TQ, TN, TP;
+		    TO = rio[WS(ios, 18)];
+		    TQ = iio[-WS(ios, 13)];
+		    TN = W[34];
+		    TP = W[35];
+		    TR = FMA(TN, TO, TP * TQ);
+		    T3j = FNMS(TP, TO, TN * TQ);
+	       }
+	       {
+		    E TU, TW, TT, TV;
+		    TU = rio[WS(ios, 10)];
+		    TW = iio[-WS(ios, 21)];
+		    TT = W[18];
+		    TV = W[19];
+		    TX = FMA(TT, TU, TV * TW);
+		    T3o = FNMS(TV, TU, TT * TW);
+	       }
+	       {
+		    E TS, T13, T5K, T5L;
+		    TS = TM + TR;
+		    T13 = TX + T12;
+		    T14 = TS + T13;
+		    T5N = TS - T13;
+		    T5K = T3i + T3j;
+		    T5L = T3o + T3p;
+		    T5M = T5K - T5L;
+		    T6E = T5K + T5L;
+	       }
+	       {
+		    E T3k, T3l, T3n, T3q;
+		    T3k = T3i - T3j;
+		    T3l = TX - T12;
+		    T3m = T3k + T3l;
+		    T4Y = T3k - T3l;
+		    T3n = TM - TR;
+		    T3q = T3o - T3p;
+		    T3r = T3n - T3q;
+		    T4Z = T3n + T3q;
+	       }
+	  }
+	  {
+	       E T1V, T3R, T20, T3S, T3Q, T3T, T26, T3M, T2b, T3N, T3L, T3O;
+	       {
+		    E T1S, T1U, T1R, T1T;
+		    T1S = rio[WS(ios, 5)];
+		    T1U = iio[-WS(ios, 26)];
+		    T1R = W[8];
+		    T1T = W[9];
+		    T1V = FMA(T1R, T1S, T1T * T1U);
+		    T3R = FNMS(T1T, T1S, T1R * T1U);
+	       }
+	       {
+		    E T1X, T1Z, T1W, T1Y;
+		    T1X = rio[WS(ios, 21)];
+		    T1Z = iio[-WS(ios, 10)];
+		    T1W = W[40];
+		    T1Y = W[41];
+		    T20 = FMA(T1W, T1X, T1Y * T1Z);
+		    T3S = FNMS(T1Y, T1X, T1W * T1Z);
+	       }
+	       T3Q = T1V - T20;
+	       T3T = T3R - T3S;
+	       {
+		    E T23, T25, T22, T24;
+		    T23 = rio[WS(ios, 29)];
+		    T25 = iio[-WS(ios, 2)];
+		    T22 = W[56];
+		    T24 = W[57];
+		    T26 = FMA(T22, T23, T24 * T25);
+		    T3M = FNMS(T24, T23, T22 * T25);
+	       }
+	       {
+		    E T28, T2a, T27, T29;
+		    T28 = rio[WS(ios, 13)];
+		    T2a = iio[-WS(ios, 18)];
+		    T27 = W[24];
+		    T29 = W[25];
+		    T2b = FMA(T27, T28, T29 * T2a);
+		    T3N = FNMS(T29, T28, T27 * T2a);
+	       }
+	       T3L = T26 - T2b;
+	       T3O = T3M - T3N;
+	       {
+		    E T21, T2c, T62, T63;
+		    T21 = T1V + T20;
+		    T2c = T26 + T2b;
+		    T2d = T21 + T2c;
+		    T5Z = T2c - T21;
+		    T62 = T3R + T3S;
+		    T63 = T3M + T3N;
+		    T64 = T62 - T63;
+		    T6K = T62 + T63;
+	       }
+	       {
+		    E T3P, T3U, T42, T43;
+		    T3P = T3L - T3O;
+		    T3U = T3Q + T3T;
+		    T3V = KP707106781 * (T3P - T3U);
+		    T57 = KP707106781 * (T3U + T3P);
+		    T42 = T3T - T3Q;
+		    T43 = T3L + T3O;
+		    T44 = KP707106781 * (T42 - T43);
+		    T5a = KP707106781 * (T42 + T43);
+	       }
+	  }
+	  {
+	       E T2G, T4c, T2L, T4d, T4e, T4f, T2R, T4i, T2W, T4j, T4h, T4k;
+	       {
+		    E T2D, T2F, T2C, T2E;
+		    T2D = rio[WS(ios, 3)];
+		    T2F = iio[-WS(ios, 28)];
+		    T2C = W[4];
+		    T2E = W[5];
+		    T2G = FMA(T2C, T2D, T2E * T2F);
+		    T4c = FNMS(T2E, T2D, T2C * T2F);
+	       }
+	       {
+		    E T2I, T2K, T2H, T2J;
+		    T2I = rio[WS(ios, 19)];
+		    T2K = iio[-WS(ios, 12)];
+		    T2H = W[36];
+		    T2J = W[37];
+		    T2L = FMA(T2H, T2I, T2J * T2K);
+		    T4d = FNMS(T2J, T2I, T2H * T2K);
+	       }
+	       T4e = T4c - T4d;
+	       T4f = T2G - T2L;
+	       {
+		    E T2O, T2Q, T2N, T2P;
+		    T2O = rio[WS(ios, 27)];
+		    T2Q = iio[-WS(ios, 4)];
+		    T2N = W[52];
+		    T2P = W[53];
+		    T2R = FMA(T2N, T2O, T2P * T2Q);
+		    T4i = FNMS(T2P, T2O, T2N * T2Q);
+	       }
+	       {
+		    E T2T, T2V, T2S, T2U;
+		    T2T = rio[WS(ios, 11)];
+		    T2V = iio[-WS(ios, 20)];
+		    T2S = W[20];
+		    T2U = W[21];
+		    T2W = FMA(T2S, T2T, T2U * T2V);
+		    T4j = FNMS(T2U, T2T, T2S * T2V);
+	       }
+	       T4h = T2R - T2W;
+	       T4k = T4i - T4j;
+	       {
+		    E T2M, T2X, T68, T69;
+		    T2M = T2G + T2L;
+		    T2X = T2R + T2W;
+		    T2Y = T2M + T2X;
+		    T6f = T2X - T2M;
+		    T68 = T4c + T4d;
+		    T69 = T4i + T4j;
+		    T6a = T68 - T69;
+		    T6P = T68 + T69;
+	       }
+	       {
+		    E T4g, T4l, T4t, T4u;
+		    T4g = T4e - T4f;
+		    T4l = T4h + T4k;
+		    T4m = KP707106781 * (T4g - T4l);
+		    T5h = KP707106781 * (T4g + T4l);
+		    T4t = T4h - T4k;
+		    T4u = T4f + T4e;
+		    T4v = KP707106781 * (T4t - T4u);
+		    T5e = KP707106781 * (T4u + T4t);
+	       }
+	  }
+	  {
+	       E T1t, T6X, T7a, T7c, T30, T7b, T70, T71;
+	       {
+		    E TH, T1s, T72, T79;
+		    TH = Tj + TG;
+		    T1s = T14 + T1r;
+		    T1t = TH + T1s;
+		    T6X = TH - T1s;
+		    T72 = T6E + T6F;
+		    T79 = T73 + T78;
+		    T7a = T72 + T79;
+		    T7c = T79 - T72;
+	       }
+	       {
+		    E T2e, T2Z, T6Y, T6Z;
+		    T2e = T1Q + T2d;
+		    T2Z = T2B + T2Y;
+		    T30 = T2e + T2Z;
+		    T7b = T2Z - T2e;
+		    T6Y = T6J + T6K;
+		    T6Z = T6O + T6P;
+		    T70 = T6Y - T6Z;
+		    T71 = T6Y + T6Z;
+	       }
+	       iio[-WS(ios, 16)] = T1t - T30;
+	       rio[WS(ios, 16)] = T71 - T7a;
+	       rio[0] = T1t + T30;
+	       iio[0] = T71 + T7a;
+	       iio[-WS(ios, 24)] = T6X - T70;
+	       rio[WS(ios, 24)] = T7b - T7c;
+	       rio[WS(ios, 8)] = T6X + T70;
+	       iio[-WS(ios, 8)] = T7b + T7c;
+	  }
+	  {
+	       E T6H, T6T, T7g, T7i, T6M, T6U, T6R, T6V;
+	       {
+		    E T6D, T6G, T7e, T7f;
+		    T6D = Tj - TG;
+		    T6G = T6E - T6F;
+		    T6H = T6D + T6G;
+		    T6T = T6D - T6G;
+		    T7e = T1r - T14;
+		    T7f = T78 - T73;
+		    T7g = T7e + T7f;
+		    T7i = T7f - T7e;
+	       }
+	       {
+		    E T6I, T6L, T6N, T6Q;
+		    T6I = T1Q - T2d;
+		    T6L = T6J - T6K;
+		    T6M = T6I + T6L;
+		    T6U = T6L - T6I;
+		    T6N = T2B - T2Y;
+		    T6Q = T6O - T6P;
+		    T6R = T6N - T6Q;
+		    T6V = T6N + T6Q;
+	       }
+	       {
+		    E T6S, T7d, T6W, T7h;
+		    T6S = KP707106781 * (T6M + T6R);
+		    iio[-WS(ios, 20)] = T6H - T6S;
+		    rio[WS(ios, 4)] = T6H + T6S;
+		    T7d = KP707106781 * (T6U + T6V);
+		    rio[WS(ios, 20)] = T7d - T7g;
+		    iio[-WS(ios, 4)] = T7d + T7g;
+		    T6W = KP707106781 * (T6U - T6V);
+		    iio[-WS(ios, 28)] = T6T - T6W;
+		    rio[WS(ios, 12)] = T6T + T6W;
+		    T7h = KP707106781 * (T6R - T6M);
+		    rio[WS(ios, 28)] = T7h - T7i;
+		    iio[-WS(ios, 12)] = T7h + T7i;
+	       }
+	  }
+	  {
+	       E T5J, T7n, T7t, T6n, T5U, T7k, T6x, T6B, T6q, T7s, T66, T6k, T6u, T6A, T6h;
+	       E T6l;
+	       {
+		    E T5O, T5T, T60, T65;
+		    T5J = T5F - T5I;
+		    T7n = T7l + T7m;
+		    T7t = T7m - T7l;
+		    T6n = T5F + T5I;
+		    T5O = T5M - T5N;
+		    T5T = T5P + T5S;
+		    T5U = KP707106781 * (T5O - T5T);
+		    T7k = KP707106781 * (T5O + T5T);
+		    {
+			 E T6v, T6w, T6o, T6p;
+			 T6v = T67 + T6a;
+			 T6w = T6e + T6f;
+			 T6x = FNMS(KP382683432, T6w, KP923879532 * T6v);
+			 T6B = FMA(KP923879532, T6w, KP382683432 * T6v);
+			 T6o = T5N + T5M;
+			 T6p = T5P - T5S;
+			 T6q = KP707106781 * (T6o + T6p);
+			 T7s = KP707106781 * (T6p - T6o);
+		    }
+		    T60 = T5Y - T5Z;
+		    T65 = T61 - T64;
+		    T66 = FMA(KP923879532, T60, KP382683432 * T65);
+		    T6k = FNMS(KP923879532, T65, KP382683432 * T60);
+		    {
+			 E T6s, T6t, T6b, T6g;
+			 T6s = T5Y + T5Z;
+			 T6t = T61 + T64;
+			 T6u = FMA(KP382683432, T6s, KP923879532 * T6t);
+			 T6A = FNMS(KP382683432, T6t, KP923879532 * T6s);
+			 T6b = T67 - T6a;
+			 T6g = T6e - T6f;
+			 T6h = FNMS(KP923879532, T6g, KP382683432 * T6b);
+			 T6l = FMA(KP382683432, T6g, KP923879532 * T6b);
+		    }
+	       }
+	       {
+		    E T5V, T6i, T7r, T7u;
+		    T5V = T5J + T5U;
+		    T6i = T66 + T6h;
+		    iio[-WS(ios, 22)] = T5V - T6i;
+		    rio[WS(ios, 6)] = T5V + T6i;
+		    T7r = T6k + T6l;
+		    T7u = T7s + T7t;
+		    rio[WS(ios, 22)] = T7r - T7u;
+		    iio[-WS(ios, 6)] = T7r + T7u;
+	       }
+	       {
+		    E T6j, T6m, T7v, T7w;
+		    T6j = T5J - T5U;
+		    T6m = T6k - T6l;
+		    iio[-WS(ios, 30)] = T6j - T6m;
+		    rio[WS(ios, 14)] = T6j + T6m;
+		    T7v = T6h - T66;
+		    T7w = T7t - T7s;
+		    rio[WS(ios, 30)] = T7v - T7w;
+		    iio[-WS(ios, 14)] = T7v + T7w;
+	       }
+	       {
+		    E T6r, T6y, T7j, T7o;
+		    T6r = T6n + T6q;
+		    T6y = T6u + T6x;
+		    iio[-WS(ios, 18)] = T6r - T6y;
+		    rio[WS(ios, 2)] = T6r + T6y;
+		    T7j = T6A + T6B;
+		    T7o = T7k + T7n;
+		    rio[WS(ios, 18)] = T7j - T7o;
+		    iio[-WS(ios, 2)] = T7j + T7o;
+	       }
+	       {
+		    E T6z, T6C, T7p, T7q;
+		    T6z = T6n - T6q;
+		    T6C = T6A - T6B;
+		    iio[-WS(ios, 26)] = T6z - T6C;
+		    rio[WS(ios, 10)] = T6z + T6C;
+		    T7p = T6x - T6u;
+		    T7q = T7n - T7k;
+		    rio[WS(ios, 26)] = T7p - T7q;
+		    iio[-WS(ios, 10)] = T7p + T7q;
+	       }
+	  }
+	  {
+	       E T3h, T4D, T7R, T7X, T3E, T7O, T4N, T4R, T46, T4A, T4G, T7W, T4K, T4Q, T4x;
+	       E T4B, T3g, T7P;
+	       T3g = KP707106781 * (T3a - T3f);
+	       T3h = T35 - T3g;
+	       T4D = T35 + T3g;
+	       T7P = KP707106781 * (T4V - T4U);
+	       T7R = T7P + T7Q;
+	       T7X = T7Q - T7P;
+	       {
+		    E T3s, T3D, T4L, T4M;
+		    T3s = FNMS(KP923879532, T3r, KP382683432 * T3m);
+		    T3D = FMA(KP382683432, T3x, KP923879532 * T3C);
+		    T3E = T3s - T3D;
+		    T7O = T3s + T3D;
+		    T4L = T4b + T4m;
+		    T4M = T4s + T4v;
+		    T4N = FNMS(KP555570233, T4M, KP831469612 * T4L);
+		    T4R = FMA(KP831469612, T4M, KP555570233 * T4L);
+	       }
+	       {
+		    E T3W, T45, T4E, T4F;
+		    T3W = T3K - T3V;
+		    T45 = T41 - T44;
+		    T46 = FMA(KP980785280, T3W, KP195090322 * T45);
+		    T4A = FNMS(KP980785280, T45, KP195090322 * T3W);
+		    T4E = FMA(KP923879532, T3m, KP382683432 * T3r);
+		    T4F = FNMS(KP923879532, T3x, KP382683432 * T3C);
+		    T4G = T4E + T4F;
+		    T7W = T4F - T4E;
+	       }
+	       {
+		    E T4I, T4J, T4n, T4w;
+		    T4I = T3K + T3V;
+		    T4J = T41 + T44;
+		    T4K = FMA(KP555570233, T4I, KP831469612 * T4J);
+		    T4Q = FNMS(KP555570233, T4J, KP831469612 * T4I);
+		    T4n = T4b - T4m;
+		    T4w = T4s - T4v;
+		    T4x = FNMS(KP980785280, T4w, KP195090322 * T4n);
+		    T4B = FMA(KP195090322, T4w, KP980785280 * T4n);
+	       }
+	       {
+		    E T3F, T4y, T7V, T7Y;
+		    T3F = T3h + T3E;
+		    T4y = T46 + T4x;
+		    iio[-WS(ios, 23)] = T3F - T4y;
+		    rio[WS(ios, 7)] = T3F + T4y;
+		    T7V = T4A + T4B;
+		    T7Y = T7W + T7X;
+		    rio[WS(ios, 23)] = T7V - T7Y;
+		    iio[-WS(ios, 7)] = T7V + T7Y;
+	       }
+	       {
+		    E T4z, T4C, T7Z, T80;
+		    T4z = T3h - T3E;
+		    T4C = T4A - T4B;
+		    iio[-WS(ios, 31)] = T4z - T4C;
+		    rio[WS(ios, 15)] = T4z + T4C;
+		    T7Z = T4x - T46;
+		    T80 = T7X - T7W;
+		    rio[WS(ios, 31)] = T7Z - T80;
+		    iio[-WS(ios, 15)] = T7Z + T80;
+	       }
+	       {
+		    E T4H, T4O, T7N, T7S;
+		    T4H = T4D + T4G;
+		    T4O = T4K + T4N;
+		    iio[-WS(ios, 19)] = T4H - T4O;
+		    rio[WS(ios, 3)] = T4H + T4O;
+		    T7N = T4Q + T4R;
+		    T7S = T7O + T7R;
+		    rio[WS(ios, 19)] = T7N - T7S;
+		    iio[-WS(ios, 3)] = T7N + T7S;
+	       }
+	       {
+		    E T4P, T4S, T7T, T7U;
+		    T4P = T4D - T4G;
+		    T4S = T4Q - T4R;
+		    iio[-WS(ios, 27)] = T4P - T4S;
+		    rio[WS(ios, 11)] = T4P + T4S;
+		    T7T = T4N - T4K;
+		    T7U = T7R - T7O;
+		    rio[WS(ios, 27)] = T7T - T7U;
+		    iio[-WS(ios, 11)] = T7T + T7U;
+	       }
+	  }
+	  {
+	       E T4X, T5p, T7D, T7J, T54, T7y, T5z, T5D, T5c, T5m, T5s, T7I, T5w, T5C, T5j;
+	       E T5n, T4W, T7z;
+	       T4W = KP707106781 * (T4U + T4V);
+	       T4X = T4T - T4W;
+	       T5p = T4T + T4W;
+	       T7z = KP707106781 * (T3a + T3f);
+	       T7D = T7z + T7C;
+	       T7J = T7C - T7z;
+	       {
+		    E T50, T53, T5x, T5y;
+		    T50 = FNMS(KP382683432, T4Z, KP923879532 * T4Y);
+		    T53 = FMA(KP923879532, T51, KP382683432 * T52);
+		    T54 = T50 - T53;
+		    T7y = T50 + T53;
+		    T5x = T5d + T5e;
+		    T5y = T5g + T5h;
+		    T5z = FNMS(KP195090322, T5y, KP980785280 * T5x);
+		    T5D = FMA(KP195090322, T5x, KP980785280 * T5y);
+	       }
+	       {
+		    E T58, T5b, T5q, T5r;
+		    T58 = T56 - T57;
+		    T5b = T59 - T5a;
+		    T5c = FMA(KP555570233, T58, KP831469612 * T5b);
+		    T5m = FNMS(KP831469612, T58, KP555570233 * T5b);
+		    T5q = FMA(KP382683432, T4Y, KP923879532 * T4Z);
+		    T5r = FNMS(KP382683432, T51, KP923879532 * T52);
+		    T5s = T5q + T5r;
+		    T7I = T5r - T5q;
+	       }
+	       {
+		    E T5u, T5v, T5f, T5i;
+		    T5u = T56 + T57;
+		    T5v = T59 + T5a;
+		    T5w = FMA(KP980785280, T5u, KP195090322 * T5v);
+		    T5C = FNMS(KP195090322, T5u, KP980785280 * T5v);
+		    T5f = T5d - T5e;
+		    T5i = T5g - T5h;
+		    T5j = FNMS(KP831469612, T5i, KP555570233 * T5f);
+		    T5n = FMA(KP831469612, T5f, KP555570233 * T5i);
+	       }
+	       {
+		    E T55, T5k, T7H, T7K;
+		    T55 = T4X + T54;
+		    T5k = T5c + T5j;
+		    iio[-WS(ios, 21)] = T55 - T5k;
+		    rio[WS(ios, 5)] = T55 + T5k;
+		    T7H = T5m + T5n;
+		    T7K = T7I + T7J;
+		    rio[WS(ios, 21)] = T7H - T7K;
+		    iio[-WS(ios, 5)] = T7H + T7K;
+	       }
+	       {
+		    E T5l, T5o, T7L, T7M;
+		    T5l = T4X - T54;
+		    T5o = T5m - T5n;
+		    iio[-WS(ios, 29)] = T5l - T5o;
+		    rio[WS(ios, 13)] = T5l + T5o;
+		    T7L = T5j - T5c;
+		    T7M = T7J - T7I;
+		    rio[WS(ios, 29)] = T7L - T7M;
+		    iio[-WS(ios, 13)] = T7L + T7M;
+	       }
+	       {
+		    E T5t, T5A, T7x, T7E;
+		    T5t = T5p + T5s;
+		    T5A = T5w + T5z;
+		    iio[-WS(ios, 17)] = T5t - T5A;
+		    rio[WS(ios, 1)] = T5t + T5A;
+		    T7x = T5C + T5D;
+		    T7E = T7y + T7D;
+		    rio[WS(ios, 17)] = T7x - T7E;
+		    iio[-WS(ios, 1)] = T7x + T7E;
+	       }
+	       {
+		    E T5B, T5E, T7F, T7G;
+		    T5B = T5p - T5s;
+		    T5E = T5C - T5D;
+		    iio[-WS(ios, 25)] = T5B - T5E;
+		    rio[WS(ios, 9)] = T5B + T5E;
+		    T7F = T5z - T5w;
+		    T7G = T7D - T7y;
+		    rio[WS(ios, 25)] = T7F - T7G;
+		    iio[-WS(ios, 9)] = T7F + T7G;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 32},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 32, "hf_32", twinstr, {340, 114, 94, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_32) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_32, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_4.c b/src/fftw3/rdft/codelets/r2hc/hf_4.c
new file mode 100644
index 0000000..f4468dd
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_4.c
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:51 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 4 -dit -name hf_4 -include hf.h */
+
+/*
+ * This function contains 22 FP additions, 12 FP multiplications,
+ * (or, 16 additions, 6 multiplications, 6 fused multiply/add),
+ * 13 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_4(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 6) {
+	  E T1, Th, Tl, Tp, T6, To, Tc, Tk;
+	  T1 = rio[0];
+	  {
+	       E Te, Tg, Td, Tf;
+	       Te = rio[WS(ios, 3)];
+	       Tg = iio[0];
+	       Td = W[4];
+	       Tf = W[5];
+	       Th = FMA(Td, Te, Tf * Tg);
+	       Tl = FNMS(Tf, Te, Td * Tg);
+	  }
+	  Tp = iio[-WS(ios, 3)];
+	  {
+	       E T3, T5, T2, T4;
+	       T3 = rio[WS(ios, 2)];
+	       T5 = iio[-WS(ios, 1)];
+	       T2 = W[2];
+	       T4 = W[3];
+	       T6 = FMA(T2, T3, T4 * T5);
+	       To = FNMS(T4, T3, T2 * T5);
+	  }
+	  {
+	       E T9, Tb, T8, Ta;
+	       T9 = rio[WS(ios, 1)];
+	       Tb = iio[-WS(ios, 2)];
+	       T8 = W[0];
+	       Ta = W[1];
+	       Tc = FMA(T8, T9, Ta * Tb);
+	       Tk = FNMS(Ta, T9, T8 * Tb);
+	  }
+	  {
+	       E T7, Ti, Tn, Tq;
+	       T7 = T1 + T6;
+	       Ti = Tc + Th;
+	       iio[-WS(ios, 2)] = T7 - Ti;
+	       rio[0] = T7 + Ti;
+	       Tn = Tk + Tl;
+	       Tq = To + Tp;
+	       rio[WS(ios, 2)] = Tn - Tq;
+	       iio[0] = Tn + Tq;
+	  }
+	  {
+	       E Tj, Tm, Tr, Ts;
+	       Tj = T1 - T6;
+	       Tm = Tk - Tl;
+	       iio[-WS(ios, 3)] = Tj - Tm;
+	       rio[WS(ios, 1)] = Tj + Tm;
+	       Tr = Th - Tc;
+	       Ts = Tp - To;
+	       rio[WS(ios, 3)] = Tr - Ts;
+	       iio[-WS(ios, 1)] = Tr + Ts;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 4},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 4, "hf_4", twinstr, {16, 6, 6, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_4) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_4, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_5.c b/src/fftw3/rdft/codelets/r2hc/hf_5.c
new file mode 100644
index 0000000..2f02461
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_5.c
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:52 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 5 -dit -name hf_5 -include hf.h */
+
+/*
+ * This function contains 40 FP additions, 28 FP multiplications,
+ * (or, 26 additions, 14 multiplications, 14 fused multiply/add),
+ * 29 stack variables, and 20 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_5.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_5.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_5.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_5(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 8) {
+	  E T1, TE, Tu, Tx, TG, TF, TB, TC, TD, Tc, Tn, To;
+	  T1 = rio[0];
+	  TE = iio[-WS(ios, 4)];
+	  {
+	       E T6, Ts, Tm, Tw, Tb, Tt, Th, Tv;
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = rio[WS(ios, 1)];
+		    T5 = iio[-WS(ios, 3)];
+		    T2 = W[0];
+		    T4 = W[1];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    Ts = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E Tj, Tl, Ti, Tk;
+		    Tj = rio[WS(ios, 3)];
+		    Tl = iio[-WS(ios, 1)];
+		    Ti = W[4];
+		    Tk = W[5];
+		    Tm = FMA(Ti, Tj, Tk * Tl);
+		    Tw = FNMS(Tk, Tj, Ti * Tl);
+	       }
+	       {
+		    E T8, Ta, T7, T9;
+		    T8 = rio[WS(ios, 4)];
+		    Ta = iio[0];
+		    T7 = W[6];
+		    T9 = W[7];
+		    Tb = FMA(T7, T8, T9 * Ta);
+		    Tt = FNMS(T9, T8, T7 * Ta);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = rio[WS(ios, 2)];
+		    Tg = iio[-WS(ios, 2)];
+		    Td = W[2];
+		    Tf = W[3];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    Tv = FNMS(Tf, Te, Td * Tg);
+	       }
+	       Tu = Ts - Tt;
+	       Tx = Tv - Tw;
+	       TG = Th - Tm;
+	       TF = Tb - T6;
+	       TB = Ts + Tt;
+	       TC = Tv + Tw;
+	       TD = TB + TC;
+	       Tc = T6 + Tb;
+	       Tn = Th + Tm;
+	       To = Tc + Tn;
+	  }
+	  rio[0] = T1 + To;
+	  iio[0] = TD + TE;
+	  {
+	       E Ty, TA, Tr, Tz, Tp, Tq;
+	       Ty = FMA(KP951056516, Tu, KP587785252 * Tx);
+	       TA = FNMS(KP587785252, Tu, KP951056516 * Tx);
+	       Tp = KP559016994 * (Tc - Tn);
+	       Tq = FNMS(KP250000000, To, T1);
+	       Tr = Tp + Tq;
+	       Tz = Tq - Tp;
+	       iio[-WS(ios, 4)] = Tr - Ty;
+	       iio[-WS(ios, 3)] = Tz + TA;
+	       rio[WS(ios, 1)] = Tr + Ty;
+	       rio[WS(ios, 2)] = Tz - TA;
+	  }
+	  {
+	       E TH, TL, TK, TM, TI, TJ;
+	       TH = FNMS(KP587785252, TG, KP951056516 * TF);
+	       TL = FMA(KP587785252, TF, KP951056516 * TG);
+	       TI = KP559016994 * (TB - TC);
+	       TJ = FNMS(KP250000000, TD, TE);
+	       TK = TI + TJ;
+	       TM = TJ - TI;
+	       rio[WS(ios, 4)] = TH - TK;
+	       iio[-WS(ios, 2)] = TL + TM;
+	       iio[-WS(ios, 1)] = TH + TK;
+	       rio[WS(ios, 3)] = TL - TM;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 5},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 5, "hf_5", twinstr, {26, 14, 14, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_5) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_5, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_6.c b/src/fftw3/rdft/codelets/r2hc/hf_6.c
new file mode 100644
index 0000000..e4c9f8d
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_6.c
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:54 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 6 -dit -name hf_6 -include hf.h */
+
+/*
+ * This function contains 46 FP additions, 28 FP multiplications,
+ * (or, 32 additions, 14 multiplications, 14 fused multiply/add),
+ * 23 stack variables, and 24 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_6.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_6.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_6.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_6(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 10) {
+	  E T7, TS, Tv, TO, Ti, TI, Tw, TC, Tt, TJ, Tx, TF;
+	  {
+	       E T1, TN, T6, TM;
+	       T1 = rio[0];
+	       TN = iio[-WS(ios, 5)];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = rio[WS(ios, 3)];
+		    T5 = iio[-WS(ios, 2)];
+		    T2 = W[4];
+		    T4 = W[5];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    TM = FNMS(T4, T3, T2 * T5);
+	       }
+	       T7 = T1 - T6;
+	       TS = TN - TM;
+	       Tv = T1 + T6;
+	       TO = TM + TN;
+	  }
+	  {
+	       E Tc, TA, Th, TB;
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = rio[WS(ios, 2)];
+		    Tb = iio[-WS(ios, 3)];
+		    T8 = W[2];
+		    Ta = W[3];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    TA = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = rio[WS(ios, 5)];
+		    Tg = iio[0];
+		    Td = W[8];
+		    Tf = W[9];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    TB = FNMS(Tf, Te, Td * Tg);
+	       }
+	       Ti = Tc - Th;
+	       TI = TA - TB;
+	       Tw = Tc + Th;
+	       TC = TA + TB;
+	  }
+	  {
+	       E Tn, TD, Ts, TE;
+	       {
+		    E Tk, Tm, Tj, Tl;
+		    Tk = rio[WS(ios, 4)];
+		    Tm = iio[-WS(ios, 1)];
+		    Tj = W[6];
+		    Tl = W[7];
+		    Tn = FMA(Tj, Tk, Tl * Tm);
+		    TD = FNMS(Tl, Tk, Tj * Tm);
+	       }
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = rio[WS(ios, 1)];
+		    Tr = iio[-WS(ios, 4)];
+		    To = W[0];
+		    Tq = W[1];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    TE = FNMS(Tq, Tp, To * Tr);
+	       }
+	       Tt = Tn - Ts;
+	       TJ = TE - TD;
+	       Tx = Tn + Ts;
+	       TF = TD + TE;
+	  }
+	  {
+	       E TK, Tu, TH, TT, TR, TU;
+	       TK = KP866025403 * (TI + TJ);
+	       Tu = Ti + Tt;
+	       TH = FNMS(KP500000000, Tu, T7);
+	       iio[-WS(ios, 3)] = T7 + Tu;
+	       rio[WS(ios, 1)] = TH + TK;
+	       iio[-WS(ios, 5)] = TH - TK;
+	       TT = KP866025403 * (Tt - Ti);
+	       TR = TJ - TI;
+	       TU = FMA(KP500000000, TR, TS);
+	       rio[WS(ios, 3)] = TR - TS;
+	       iio[-WS(ios, 1)] = TT + TU;
+	       rio[WS(ios, 5)] = TT - TU;
+	  }
+	  {
+	       E TG, Ty, Tz, TP, TL, TQ;
+	       TG = KP866025403 * (TC - TF);
+	       Ty = Tw + Tx;
+	       Tz = FNMS(KP500000000, Ty, Tv);
+	       rio[0] = Tv + Ty;
+	       iio[-WS(ios, 4)] = Tz + TG;
+	       rio[WS(ios, 2)] = Tz - TG;
+	       TP = KP866025403 * (Tw - Tx);
+	       TL = TC + TF;
+	       TQ = FNMS(KP500000000, TL, TO);
+	       iio[0] = TL + TO;
+	       iio[-WS(ios, 2)] = TP + TQ;
+	       rio[WS(ios, 4)] = TP - TQ;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 6},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 6, "hf_6", twinstr, {32, 14, 14, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_6) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_6, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_64.c b/src/fftw3/rdft/codelets/r2hc/hf_64.c
new file mode 100644
index 0000000..3e99d63
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_64.c
@@ -0,0 +1,2001 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:11 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 64 -dit -name hf_64 -include hf.h */
+
+/*
+ * This function contains 1038 FP additions, 500 FP multiplications,
+ * (or, 808 additions, 270 multiplications, 230 fused multiply/add),
+ * 176 stack variables, and 256 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_64(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP471396736, +0.471396736825997648556387625905254377657460319);
+     DK(KP881921264, +0.881921264348355029712756863660388349508442621);
+     DK(KP290284677, +0.290284677254462367636192375817395274691476278);
+     DK(KP956940335, +0.956940335732208864935797886980269969482849206);
+     DK(KP634393284, +0.634393284163645498215171613225493370675687095);
+     DK(KP773010453, +0.773010453362736960810906609758469800971041293);
+     DK(KP098017140, +0.098017140329560601994195563888641845861136673);
+     DK(KP995184726, +0.995184726672196886244836953109479921575474869);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 126) {
+	  E Tj, TcL, ThT, Tin, T6b, Taz, TgT, Thn, TG, Thm, TcO, TgO, T6m, ThQ, TaC;
+	  E Tim, T14, Tfq, T6y, T9O, TaG, Tc0, TcU, TeE, T1r, Tfr, T6J, T9P, TaJ, Tc1;
+	  E TcZ, TeF, T1Q, T2d, Tfx, Tfu, Tfv, Tfw, T6Q, TaM, Tdb, TeJ, T71, TaQ, T7a;
+	  E TaN, Td6, TeI, T77, TaP, T2B, T2Y, Tfz, TfA, TfB, TfC, T7h, TaW, Tdm, TeM;
+	  E T7s, TaU, T7B, TaX, Tdh, TeL, T7y, TaT, T3M, TfL, TdL, TeQ, TfI, Tgt, T7K;
+	  E Tb2, T7V, Tbe, T8s, Tb3, Tdu, TeT, T8p, Tbd, T5j, TfR, Tec, Tf0, TfY, Tgy;
+	  E T8D, Tbl, T8O, Tbx, T9l, Tbm, TdV, TeX, T9i, Tbw, T64, TfZ, Te5, Ted, TfU;
+	  E Tgz, T90, T9o, T9b, T9n, Tbt, Tbz, Te0, Tee, Tbq, TbA, T4x, TfJ, TdE, TdM;
+	  E TfO, Tgu, T87, T8v, T8i, T8u, Tba, Tbg, Tdz, TdN, Tb7, Tbh;
+	  {
+	       E T1, TgR, T6, TgQ, Tc, T68, Th, T69;
+	       T1 = rio[0];
+	       TgR = iio[-WS(ios, 63)];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = rio[WS(ios, 32)];
+		    T5 = iio[-WS(ios, 31)];
+		    T2 = W[62];
+		    T4 = W[63];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    TgQ = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = rio[WS(ios, 16)];
+		    Tb = iio[-WS(ios, 47)];
+		    T8 = W[30];
+		    Ta = W[31];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    T68 = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = rio[WS(ios, 48)];
+		    Tg = iio[-WS(ios, 15)];
+		    Td = W[94];
+		    Tf = W[95];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    T69 = FNMS(Tf, Te, Td * Tg);
+	       }
+	       {
+		    E T7, Ti, ThR, ThS;
+		    T7 = T1 + T6;
+		    Ti = Tc + Th;
+		    Tj = T7 + Ti;
+		    TcL = T7 - Ti;
+		    ThR = TgR - TgQ;
+		    ThS = Tc - Th;
+		    ThT = ThR - ThS;
+		    Tin = ThS + ThR;
+	       }
+	       {
+		    E T67, T6a, TgP, TgS;
+		    T67 = T1 - T6;
+		    T6a = T68 - T69;
+		    T6b = T67 - T6a;
+		    Taz = T67 + T6a;
+		    TgP = T68 + T69;
+		    TgS = TgQ + TgR;
+		    TgT = TgP + TgS;
+		    Thn = TgS - TgP;
+	       }
+	  }
+	  {
+	       E To, T6c, Tt, T6d, T6e, T6f, Tz, T6i, TE, T6j, T6h, T6k;
+	       {
+		    E Tl, Tn, Tk, Tm;
+		    Tl = rio[WS(ios, 8)];
+		    Tn = iio[-WS(ios, 55)];
+		    Tk = W[14];
+		    Tm = W[15];
+		    To = FMA(Tk, Tl, Tm * Tn);
+		    T6c = FNMS(Tm, Tl, Tk * Tn);
+	       }
+	       {
+		    E Tq, Ts, Tp, Tr;
+		    Tq = rio[WS(ios, 40)];
+		    Ts = iio[-WS(ios, 23)];
+		    Tp = W[78];
+		    Tr = W[79];
+		    Tt = FMA(Tp, Tq, Tr * Ts);
+		    T6d = FNMS(Tr, Tq, Tp * Ts);
+	       }
+	       T6e = T6c - T6d;
+	       T6f = To - Tt;
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = rio[WS(ios, 56)];
+		    Ty = iio[-WS(ios, 7)];
+		    Tv = W[110];
+		    Tx = W[111];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T6i = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = rio[WS(ios, 24)];
+		    TD = iio[-WS(ios, 39)];
+		    TA = W[46];
+		    TC = W[47];
+		    TE = FMA(TA, TB, TC * TD);
+		    T6j = FNMS(TC, TB, TA * TD);
+	       }
+	       T6h = Tz - TE;
+	       T6k = T6i - T6j;
+	       {
+		    E Tu, TF, TcM, TcN;
+		    Tu = To + Tt;
+		    TF = Tz + TE;
+		    TG = Tu + TF;
+		    Thm = TF - Tu;
+		    TcM = T6c + T6d;
+		    TcN = T6i + T6j;
+		    TcO = TcM - TcN;
+		    TgO = TcM + TcN;
+	       }
+	       {
+		    E T6g, T6l, TaA, TaB;
+		    T6g = T6e - T6f;
+		    T6l = T6h + T6k;
+		    T6m = KP707106781 * (T6g - T6l);
+		    ThQ = KP707106781 * (T6g + T6l);
+		    TaA = T6f + T6e;
+		    TaB = T6h - T6k;
+		    TaC = KP707106781 * (TaA + TaB);
+		    Tim = KP707106781 * (TaB - TaA);
+	       }
+	  }
+	  {
+	       E TS, TcQ, T6q, T6t, T13, TcR, T6r, T6w, T6s, T6x;
+	       {
+		    E TM, T6o, TR, T6p;
+		    {
+			 E TJ, TL, TI, TK;
+			 TJ = rio[WS(ios, 4)];
+			 TL = iio[-WS(ios, 59)];
+			 TI = W[6];
+			 TK = W[7];
+			 TM = FMA(TI, TJ, TK * TL);
+			 T6o = FNMS(TK, TJ, TI * TL);
+		    }
+		    {
+			 E TO, TQ, TN, TP;
+			 TO = rio[WS(ios, 36)];
+			 TQ = iio[-WS(ios, 27)];
+			 TN = W[70];
+			 TP = W[71];
+			 TR = FMA(TN, TO, TP * TQ);
+			 T6p = FNMS(TP, TO, TN * TQ);
+		    }
+		    TS = TM + TR;
+		    TcQ = T6o + T6p;
+		    T6q = T6o - T6p;
+		    T6t = TM - TR;
+	       }
+	       {
+		    E TX, T6u, T12, T6v;
+		    {
+			 E TU, TW, TT, TV;
+			 TU = rio[WS(ios, 20)];
+			 TW = iio[-WS(ios, 43)];
+			 TT = W[38];
+			 TV = W[39];
+			 TX = FMA(TT, TU, TV * TW);
+			 T6u = FNMS(TV, TU, TT * TW);
+		    }
+		    {
+			 E TZ, T11, TY, T10;
+			 TZ = rio[WS(ios, 52)];
+			 T11 = iio[-WS(ios, 11)];
+			 TY = W[102];
+			 T10 = W[103];
+			 T12 = FMA(TY, TZ, T10 * T11);
+			 T6v = FNMS(T10, TZ, TY * T11);
+		    }
+		    T13 = TX + T12;
+		    TcR = T6u + T6v;
+		    T6r = TX - T12;
+		    T6w = T6u - T6v;
+	       }
+	       T14 = TS + T13;
+	       Tfq = TcQ + TcR;
+	       T6s = T6q + T6r;
+	       T6x = T6t - T6w;
+	       T6y = FNMS(KP923879532, T6x, KP382683432 * T6s);
+	       T9O = FMA(KP923879532, T6s, KP382683432 * T6x);
+	       {
+		    E TaE, TaF, TcS, TcT;
+		    TaE = T6q - T6r;
+		    TaF = T6t + T6w;
+		    TaG = FNMS(KP382683432, TaF, KP923879532 * TaE);
+		    Tc0 = FMA(KP382683432, TaE, KP923879532 * TaF);
+		    TcS = TcQ - TcR;
+		    TcT = TS - T13;
+		    TcU = TcS - TcT;
+		    TeE = TcT + TcS;
+	       }
+	  }
+	  {
+	       E T1f, TcW, T6B, T6E, T1q, TcX, T6C, T6H, T6D, T6I;
+	       {
+		    E T19, T6z, T1e, T6A;
+		    {
+			 E T16, T18, T15, T17;
+			 T16 = rio[WS(ios, 60)];
+			 T18 = iio[-WS(ios, 3)];
+			 T15 = W[118];
+			 T17 = W[119];
+			 T19 = FMA(T15, T16, T17 * T18);
+			 T6z = FNMS(T17, T16, T15 * T18);
+		    }
+		    {
+			 E T1b, T1d, T1a, T1c;
+			 T1b = rio[WS(ios, 28)];
+			 T1d = iio[-WS(ios, 35)];
+			 T1a = W[54];
+			 T1c = W[55];
+			 T1e = FMA(T1a, T1b, T1c * T1d);
+			 T6A = FNMS(T1c, T1b, T1a * T1d);
+		    }
+		    T1f = T19 + T1e;
+		    TcW = T6z + T6A;
+		    T6B = T6z - T6A;
+		    T6E = T19 - T1e;
+	       }
+	       {
+		    E T1k, T6F, T1p, T6G;
+		    {
+			 E T1h, T1j, T1g, T1i;
+			 T1h = rio[WS(ios, 12)];
+			 T1j = iio[-WS(ios, 51)];
+			 T1g = W[22];
+			 T1i = W[23];
+			 T1k = FMA(T1g, T1h, T1i * T1j);
+			 T6F = FNMS(T1i, T1h, T1g * T1j);
+		    }
+		    {
+			 E T1m, T1o, T1l, T1n;
+			 T1m = rio[WS(ios, 44)];
+			 T1o = iio[-WS(ios, 19)];
+			 T1l = W[86];
+			 T1n = W[87];
+			 T1p = FMA(T1l, T1m, T1n * T1o);
+			 T6G = FNMS(T1n, T1m, T1l * T1o);
+		    }
+		    T1q = T1k + T1p;
+		    TcX = T6F + T6G;
+		    T6C = T1k - T1p;
+		    T6H = T6F - T6G;
+	       }
+	       T1r = T1f + T1q;
+	       Tfr = TcW + TcX;
+	       T6D = T6B + T6C;
+	       T6I = T6E - T6H;
+	       T6J = FMA(KP382683432, T6D, KP923879532 * T6I);
+	       T9P = FNMS(KP923879532, T6D, KP382683432 * T6I);
+	       {
+		    E TaH, TaI, TcV, TcY;
+		    TaH = T6B - T6C;
+		    TaI = T6E + T6H;
+		    TaJ = FMA(KP923879532, TaH, KP382683432 * TaI);
+		    Tc1 = FNMS(KP382683432, TaH, KP923879532 * TaI);
+		    TcV = T1f - T1q;
+		    TcY = TcW - TcX;
+		    TcZ = TcV + TcY;
+		    TeF = TcV - TcY;
+	       }
+	  }
+	  {
+	       E T1y, T6M, T1D, T6N, T1E, Td2, T1J, T74, T1O, T75, T1P, Td3, T21, Td8, T6W;
+	       E T6Z, T2c, Td9, T6R, T6U;
+	       {
+		    E T1v, T1x, T1u, T1w;
+		    T1v = rio[WS(ios, 2)];
+		    T1x = iio[-WS(ios, 61)];
+		    T1u = W[2];
+		    T1w = W[3];
+		    T1y = FMA(T1u, T1v, T1w * T1x);
+		    T6M = FNMS(T1w, T1v, T1u * T1x);
+	       }
+	       {
+		    E T1A, T1C, T1z, T1B;
+		    T1A = rio[WS(ios, 34)];
+		    T1C = iio[-WS(ios, 29)];
+		    T1z = W[66];
+		    T1B = W[67];
+		    T1D = FMA(T1z, T1A, T1B * T1C);
+		    T6N = FNMS(T1B, T1A, T1z * T1C);
+	       }
+	       T1E = T1y + T1D;
+	       Td2 = T6M + T6N;
+	       {
+		    E T1G, T1I, T1F, T1H;
+		    T1G = rio[WS(ios, 18)];
+		    T1I = iio[-WS(ios, 45)];
+		    T1F = W[34];
+		    T1H = W[35];
+		    T1J = FMA(T1F, T1G, T1H * T1I);
+		    T74 = FNMS(T1H, T1G, T1F * T1I);
+	       }
+	       {
+		    E T1L, T1N, T1K, T1M;
+		    T1L = rio[WS(ios, 50)];
+		    T1N = iio[-WS(ios, 13)];
+		    T1K = W[98];
+		    T1M = W[99];
+		    T1O = FMA(T1K, T1L, T1M * T1N);
+		    T75 = FNMS(T1M, T1L, T1K * T1N);
+	       }
+	       T1P = T1J + T1O;
+	       Td3 = T74 + T75;
+	       {
+		    E T1V, T6X, T20, T6Y;
+		    {
+			 E T1S, T1U, T1R, T1T;
+			 T1S = rio[WS(ios, 10)];
+			 T1U = iio[-WS(ios, 53)];
+			 T1R = W[18];
+			 T1T = W[19];
+			 T1V = FMA(T1R, T1S, T1T * T1U);
+			 T6X = FNMS(T1T, T1S, T1R * T1U);
+		    }
+		    {
+			 E T1X, T1Z, T1W, T1Y;
+			 T1X = rio[WS(ios, 42)];
+			 T1Z = iio[-WS(ios, 21)];
+			 T1W = W[82];
+			 T1Y = W[83];
+			 T20 = FMA(T1W, T1X, T1Y * T1Z);
+			 T6Y = FNMS(T1Y, T1X, T1W * T1Z);
+		    }
+		    T21 = T1V + T20;
+		    Td8 = T6X + T6Y;
+		    T6W = T1V - T20;
+		    T6Z = T6X - T6Y;
+	       }
+	       {
+		    E T26, T6S, T2b, T6T;
+		    {
+			 E T23, T25, T22, T24;
+			 T23 = rio[WS(ios, 58)];
+			 T25 = iio[-WS(ios, 5)];
+			 T22 = W[114];
+			 T24 = W[115];
+			 T26 = FMA(T22, T23, T24 * T25);
+			 T6S = FNMS(T24, T23, T22 * T25);
+		    }
+		    {
+			 E T28, T2a, T27, T29;
+			 T28 = rio[WS(ios, 26)];
+			 T2a = iio[-WS(ios, 37)];
+			 T27 = W[50];
+			 T29 = W[51];
+			 T2b = FMA(T27, T28, T29 * T2a);
+			 T6T = FNMS(T29, T28, T27 * T2a);
+		    }
+		    T2c = T26 + T2b;
+		    Td9 = T6S + T6T;
+		    T6R = T26 - T2b;
+		    T6U = T6S - T6T;
+	       }
+	       T1Q = T1E + T1P;
+	       T2d = T21 + T2c;
+	       Tfx = T1Q - T2d;
+	       Tfu = Td2 + Td3;
+	       Tfv = Td8 + Td9;
+	       Tfw = Tfu - Tfv;
+	       {
+		    E T6O, T6P, Td7, Tda;
+		    T6O = T6M - T6N;
+		    T6P = T1J - T1O;
+		    T6Q = T6O + T6P;
+		    TaM = T6O - T6P;
+		    Td7 = T1E - T1P;
+		    Tda = Td8 - Td9;
+		    Tdb = Td7 - Tda;
+		    TeJ = Td7 + Tda;
+	       }
+	       {
+		    E T6V, T70, T78, T79;
+		    T6V = T6R - T6U;
+		    T70 = T6W + T6Z;
+		    T71 = KP707106781 * (T6V - T70);
+		    TaQ = KP707106781 * (T70 + T6V);
+		    T78 = T6Z - T6W;
+		    T79 = T6R + T6U;
+		    T7a = KP707106781 * (T78 - T79);
+		    TaN = KP707106781 * (T78 + T79);
+	       }
+	       {
+		    E Td4, Td5, T73, T76;
+		    Td4 = Td2 - Td3;
+		    Td5 = T2c - T21;
+		    Td6 = Td4 - Td5;
+		    TeI = Td4 + Td5;
+		    T73 = T1y - T1D;
+		    T76 = T74 - T75;
+		    T77 = T73 - T76;
+		    TaP = T73 + T76;
+	       }
+	  }
+	  {
+	       E T2j, T7d, T2o, T7e, T2p, Tdd, T2u, T7v, T2z, T7w, T2A, Tde, T2M, Tdj, T7n;
+	       E T7q, T2X, Tdk, T7i, T7l;
+	       {
+		    E T2g, T2i, T2f, T2h;
+		    T2g = rio[WS(ios, 62)];
+		    T2i = iio[-WS(ios, 1)];
+		    T2f = W[122];
+		    T2h = W[123];
+		    T2j = FMA(T2f, T2g, T2h * T2i);
+		    T7d = FNMS(T2h, T2g, T2f * T2i);
+	       }
+	       {
+		    E T2l, T2n, T2k, T2m;
+		    T2l = rio[WS(ios, 30)];
+		    T2n = iio[-WS(ios, 33)];
+		    T2k = W[58];
+		    T2m = W[59];
+		    T2o = FMA(T2k, T2l, T2m * T2n);
+		    T7e = FNMS(T2m, T2l, T2k * T2n);
+	       }
+	       T2p = T2j + T2o;
+	       Tdd = T7d + T7e;
+	       {
+		    E T2r, T2t, T2q, T2s;
+		    T2r = rio[WS(ios, 14)];
+		    T2t = iio[-WS(ios, 49)];
+		    T2q = W[26];
+		    T2s = W[27];
+		    T2u = FMA(T2q, T2r, T2s * T2t);
+		    T7v = FNMS(T2s, T2r, T2q * T2t);
+	       }
+	       {
+		    E T2w, T2y, T2v, T2x;
+		    T2w = rio[WS(ios, 46)];
+		    T2y = iio[-WS(ios, 17)];
+		    T2v = W[90];
+		    T2x = W[91];
+		    T2z = FMA(T2v, T2w, T2x * T2y);
+		    T7w = FNMS(T2x, T2w, T2v * T2y);
+	       }
+	       T2A = T2u + T2z;
+	       Tde = T7v + T7w;
+	       {
+		    E T2G, T7o, T2L, T7p;
+		    {
+			 E T2D, T2F, T2C, T2E;
+			 T2D = rio[WS(ios, 6)];
+			 T2F = iio[-WS(ios, 57)];
+			 T2C = W[10];
+			 T2E = W[11];
+			 T2G = FMA(T2C, T2D, T2E * T2F);
+			 T7o = FNMS(T2E, T2D, T2C * T2F);
+		    }
+		    {
+			 E T2I, T2K, T2H, T2J;
+			 T2I = rio[WS(ios, 38)];
+			 T2K = iio[-WS(ios, 25)];
+			 T2H = W[74];
+			 T2J = W[75];
+			 T2L = FMA(T2H, T2I, T2J * T2K);
+			 T7p = FNMS(T2J, T2I, T2H * T2K);
+		    }
+		    T2M = T2G + T2L;
+		    Tdj = T7o + T7p;
+		    T7n = T2G - T2L;
+		    T7q = T7o - T7p;
+	       }
+	       {
+		    E T2R, T7j, T2W, T7k;
+		    {
+			 E T2O, T2Q, T2N, T2P;
+			 T2O = rio[WS(ios, 54)];
+			 T2Q = iio[-WS(ios, 9)];
+			 T2N = W[106];
+			 T2P = W[107];
+			 T2R = FMA(T2N, T2O, T2P * T2Q);
+			 T7j = FNMS(T2P, T2O, T2N * T2Q);
+		    }
+		    {
+			 E T2T, T2V, T2S, T2U;
+			 T2T = rio[WS(ios, 22)];
+			 T2V = iio[-WS(ios, 41)];
+			 T2S = W[42];
+			 T2U = W[43];
+			 T2W = FMA(T2S, T2T, T2U * T2V);
+			 T7k = FNMS(T2U, T2T, T2S * T2V);
+		    }
+		    T2X = T2R + T2W;
+		    Tdk = T7j + T7k;
+		    T7i = T2R - T2W;
+		    T7l = T7j - T7k;
+	       }
+	       T2B = T2p + T2A;
+	       T2Y = T2M + T2X;
+	       Tfz = T2B - T2Y;
+	       TfA = Tdd + Tde;
+	       TfB = Tdj + Tdk;
+	       TfC = TfA - TfB;
+	       {
+		    E T7f, T7g, Tdi, Tdl;
+		    T7f = T7d - T7e;
+		    T7g = T2u - T2z;
+		    T7h = T7f + T7g;
+		    TaW = T7f - T7g;
+		    Tdi = T2p - T2A;
+		    Tdl = Tdj - Tdk;
+		    Tdm = Tdi - Tdl;
+		    TeM = Tdi + Tdl;
+	       }
+	       {
+		    E T7m, T7r, T7z, T7A;
+		    T7m = T7i - T7l;
+		    T7r = T7n + T7q;
+		    T7s = KP707106781 * (T7m - T7r);
+		    TaU = KP707106781 * (T7r + T7m);
+		    T7z = T7q - T7n;
+		    T7A = T7i + T7l;
+		    T7B = KP707106781 * (T7z - T7A);
+		    TaX = KP707106781 * (T7z + T7A);
+	       }
+	       {
+		    E Tdf, Tdg, T7u, T7x;
+		    Tdf = Tdd - Tde;
+		    Tdg = T2X - T2M;
+		    Tdh = Tdf - Tdg;
+		    TeL = Tdf + Tdg;
+		    T7u = T2j - T2o;
+		    T7x = T7v - T7w;
+		    T7y = T7u - T7x;
+		    TaT = T7u + T7x;
+	       }
+	  }
+	  {
+	       E T36, T7G, T3b, T7H, T3c, Tdq, T3h, T8m, T3m, T8n, T3n, Tdr, T3z, TdI, T7Q;
+	       E T7T, T3K, TdJ, T7L, T7O;
+	       {
+		    E T33, T35, T32, T34;
+		    T33 = rio[WS(ios, 1)];
+		    T35 = iio[-WS(ios, 62)];
+		    T32 = W[0];
+		    T34 = W[1];
+		    T36 = FMA(T32, T33, T34 * T35);
+		    T7G = FNMS(T34, T33, T32 * T35);
+	       }
+	       {
+		    E T38, T3a, T37, T39;
+		    T38 = rio[WS(ios, 33)];
+		    T3a = iio[-WS(ios, 30)];
+		    T37 = W[64];
+		    T39 = W[65];
+		    T3b = FMA(T37, T38, T39 * T3a);
+		    T7H = FNMS(T39, T38, T37 * T3a);
+	       }
+	       T3c = T36 + T3b;
+	       Tdq = T7G + T7H;
+	       {
+		    E T3e, T3g, T3d, T3f;
+		    T3e = rio[WS(ios, 17)];
+		    T3g = iio[-WS(ios, 46)];
+		    T3d = W[32];
+		    T3f = W[33];
+		    T3h = FMA(T3d, T3e, T3f * T3g);
+		    T8m = FNMS(T3f, T3e, T3d * T3g);
+	       }
+	       {
+		    E T3j, T3l, T3i, T3k;
+		    T3j = rio[WS(ios, 49)];
+		    T3l = iio[-WS(ios, 14)];
+		    T3i = W[96];
+		    T3k = W[97];
+		    T3m = FMA(T3i, T3j, T3k * T3l);
+		    T8n = FNMS(T3k, T3j, T3i * T3l);
+	       }
+	       T3n = T3h + T3m;
+	       Tdr = T8m + T8n;
+	       {
+		    E T3t, T7R, T3y, T7S;
+		    {
+			 E T3q, T3s, T3p, T3r;
+			 T3q = rio[WS(ios, 9)];
+			 T3s = iio[-WS(ios, 54)];
+			 T3p = W[16];
+			 T3r = W[17];
+			 T3t = FMA(T3p, T3q, T3r * T3s);
+			 T7R = FNMS(T3r, T3q, T3p * T3s);
+		    }
+		    {
+			 E T3v, T3x, T3u, T3w;
+			 T3v = rio[WS(ios, 41)];
+			 T3x = iio[-WS(ios, 22)];
+			 T3u = W[80];
+			 T3w = W[81];
+			 T3y = FMA(T3u, T3v, T3w * T3x);
+			 T7S = FNMS(T3w, T3v, T3u * T3x);
+		    }
+		    T3z = T3t + T3y;
+		    TdI = T7R + T7S;
+		    T7Q = T3t - T3y;
+		    T7T = T7R - T7S;
+	       }
+	       {
+		    E T3E, T7M, T3J, T7N;
+		    {
+			 E T3B, T3D, T3A, T3C;
+			 T3B = rio[WS(ios, 57)];
+			 T3D = iio[-WS(ios, 6)];
+			 T3A = W[112];
+			 T3C = W[113];
+			 T3E = FMA(T3A, T3B, T3C * T3D);
+			 T7M = FNMS(T3C, T3B, T3A * T3D);
+		    }
+		    {
+			 E T3G, T3I, T3F, T3H;
+			 T3G = rio[WS(ios, 25)];
+			 T3I = iio[-WS(ios, 38)];
+			 T3F = W[48];
+			 T3H = W[49];
+			 T3J = FMA(T3F, T3G, T3H * T3I);
+			 T7N = FNMS(T3H, T3G, T3F * T3I);
+		    }
+		    T3K = T3E + T3J;
+		    TdJ = T7M + T7N;
+		    T7L = T3E - T3J;
+		    T7O = T7M - T7N;
+	       }
+	       {
+		    E T3o, T3L, TdH, TdK;
+		    T3o = T3c + T3n;
+		    T3L = T3z + T3K;
+		    T3M = T3o + T3L;
+		    TfL = T3o - T3L;
+		    TdH = T3c - T3n;
+		    TdK = TdI - TdJ;
+		    TdL = TdH - TdK;
+		    TeQ = TdH + TdK;
+	       }
+	       {
+		    E TfG, TfH, T7I, T7J;
+		    TfG = Tdq + Tdr;
+		    TfH = TdI + TdJ;
+		    TfI = TfG - TfH;
+		    Tgt = TfG + TfH;
+		    T7I = T7G - T7H;
+		    T7J = T3h - T3m;
+		    T7K = T7I + T7J;
+		    Tb2 = T7I - T7J;
+	       }
+	       {
+		    E T7P, T7U, T8q, T8r;
+		    T7P = T7L - T7O;
+		    T7U = T7Q + T7T;
+		    T7V = KP707106781 * (T7P - T7U);
+		    Tbe = KP707106781 * (T7U + T7P);
+		    T8q = T7T - T7Q;
+		    T8r = T7L + T7O;
+		    T8s = KP707106781 * (T8q - T8r);
+		    Tb3 = KP707106781 * (T8q + T8r);
+	       }
+	       {
+		    E Tds, Tdt, T8l, T8o;
+		    Tds = Tdq - Tdr;
+		    Tdt = T3K - T3z;
+		    Tdu = Tds - Tdt;
+		    TeT = Tds + Tdt;
+		    T8l = T36 - T3b;
+		    T8o = T8m - T8n;
+		    T8p = T8l - T8o;
+		    Tbd = T8l + T8o;
+	       }
+	  }
+	  {
+	       E T4D, T9e, T4I, T9f, T4J, Te8, T4O, T8A, T4T, T8B, T4U, Te9, T56, TdS, T8G;
+	       E T8H, T5h, TdT, T8J, T8M;
+	       {
+		    E T4A, T4C, T4z, T4B;
+		    T4A = rio[WS(ios, 63)];
+		    T4C = iio[0];
+		    T4z = W[124];
+		    T4B = W[125];
+		    T4D = FMA(T4z, T4A, T4B * T4C);
+		    T9e = FNMS(T4B, T4A, T4z * T4C);
+	       }
+	       {
+		    E T4F, T4H, T4E, T4G;
+		    T4F = rio[WS(ios, 31)];
+		    T4H = iio[-WS(ios, 32)];
+		    T4E = W[60];
+		    T4G = W[61];
+		    T4I = FMA(T4E, T4F, T4G * T4H);
+		    T9f = FNMS(T4G, T4F, T4E * T4H);
+	       }
+	       T4J = T4D + T4I;
+	       Te8 = T9e + T9f;
+	       {
+		    E T4L, T4N, T4K, T4M;
+		    T4L = rio[WS(ios, 15)];
+		    T4N = iio[-WS(ios, 48)];
+		    T4K = W[28];
+		    T4M = W[29];
+		    T4O = FMA(T4K, T4L, T4M * T4N);
+		    T8A = FNMS(T4M, T4L, T4K * T4N);
+	       }
+	       {
+		    E T4Q, T4S, T4P, T4R;
+		    T4Q = rio[WS(ios, 47)];
+		    T4S = iio[-WS(ios, 16)];
+		    T4P = W[92];
+		    T4R = W[93];
+		    T4T = FMA(T4P, T4Q, T4R * T4S);
+		    T8B = FNMS(T4R, T4Q, T4P * T4S);
+	       }
+	       T4U = T4O + T4T;
+	       Te9 = T8A + T8B;
+	       {
+		    E T50, T8E, T55, T8F;
+		    {
+			 E T4X, T4Z, T4W, T4Y;
+			 T4X = rio[WS(ios, 7)];
+			 T4Z = iio[-WS(ios, 56)];
+			 T4W = W[12];
+			 T4Y = W[13];
+			 T50 = FMA(T4W, T4X, T4Y * T4Z);
+			 T8E = FNMS(T4Y, T4X, T4W * T4Z);
+		    }
+		    {
+			 E T52, T54, T51, T53;
+			 T52 = rio[WS(ios, 39)];
+			 T54 = iio[-WS(ios, 24)];
+			 T51 = W[76];
+			 T53 = W[77];
+			 T55 = FMA(T51, T52, T53 * T54);
+			 T8F = FNMS(T53, T52, T51 * T54);
+		    }
+		    T56 = T50 + T55;
+		    TdS = T8E + T8F;
+		    T8G = T8E - T8F;
+		    T8H = T50 - T55;
+	       }
+	       {
+		    E T5b, T8K, T5g, T8L;
+		    {
+			 E T58, T5a, T57, T59;
+			 T58 = rio[WS(ios, 55)];
+			 T5a = iio[-WS(ios, 8)];
+			 T57 = W[108];
+			 T59 = W[109];
+			 T5b = FMA(T57, T58, T59 * T5a);
+			 T8K = FNMS(T59, T58, T57 * T5a);
+		    }
+		    {
+			 E T5d, T5f, T5c, T5e;
+			 T5d = rio[WS(ios, 23)];
+			 T5f = iio[-WS(ios, 40)];
+			 T5c = W[44];
+			 T5e = W[45];
+			 T5g = FMA(T5c, T5d, T5e * T5f);
+			 T8L = FNMS(T5e, T5d, T5c * T5f);
+		    }
+		    T5h = T5b + T5g;
+		    TdT = T8K + T8L;
+		    T8J = T5b - T5g;
+		    T8M = T8K - T8L;
+	       }
+	       {
+		    E T4V, T5i, Tea, Teb;
+		    T4V = T4J + T4U;
+		    T5i = T56 + T5h;
+		    T5j = T4V + T5i;
+		    TfR = T4V - T5i;
+		    Tea = Te8 - Te9;
+		    Teb = T5h - T56;
+		    Tec = Tea - Teb;
+		    Tf0 = Tea + Teb;
+	       }
+	       {
+		    E TfW, TfX, T8z, T8C;
+		    TfW = Te8 + Te9;
+		    TfX = TdS + TdT;
+		    TfY = TfW - TfX;
+		    Tgy = TfW + TfX;
+		    T8z = T4D - T4I;
+		    T8C = T8A - T8B;
+		    T8D = T8z - T8C;
+		    Tbl = T8z + T8C;
+	       }
+	       {
+		    E T8I, T8N, T9j, T9k;
+		    T8I = T8G - T8H;
+		    T8N = T8J + T8M;
+		    T8O = KP707106781 * (T8I - T8N);
+		    Tbx = KP707106781 * (T8I + T8N);
+		    T9j = T8J - T8M;
+		    T9k = T8H + T8G;
+		    T9l = KP707106781 * (T9j - T9k);
+		    Tbm = KP707106781 * (T9k + T9j);
+	       }
+	       {
+		    E TdR, TdU, T9g, T9h;
+		    TdR = T4J - T4U;
+		    TdU = TdS - TdT;
+		    TdV = TdR - TdU;
+		    TeX = TdR + TdU;
+		    T9g = T9e - T9f;
+		    T9h = T4O - T4T;
+		    T9i = T9g + T9h;
+		    Tbw = T9g - T9h;
+	       }
+	  }
+	  {
+	       E T5u, TdW, T8S, T8V, T62, Te3, T94, T99, T5F, TdX, T8T, T8Y, T5R, Te2, T93;
+	       E T96;
+	       {
+		    E T5o, T8Q, T5t, T8R;
+		    {
+			 E T5l, T5n, T5k, T5m;
+			 T5l = rio[WS(ios, 3)];
+			 T5n = iio[-WS(ios, 60)];
+			 T5k = W[4];
+			 T5m = W[5];
+			 T5o = FMA(T5k, T5l, T5m * T5n);
+			 T8Q = FNMS(T5m, T5l, T5k * T5n);
+		    }
+		    {
+			 E T5q, T5s, T5p, T5r;
+			 T5q = rio[WS(ios, 35)];
+			 T5s = iio[-WS(ios, 28)];
+			 T5p = W[68];
+			 T5r = W[69];
+			 T5t = FMA(T5p, T5q, T5r * T5s);
+			 T8R = FNMS(T5r, T5q, T5p * T5s);
+		    }
+		    T5u = T5o + T5t;
+		    TdW = T8Q + T8R;
+		    T8S = T8Q - T8R;
+		    T8V = T5o - T5t;
+	       }
+	       {
+		    E T5W, T97, T61, T98;
+		    {
+			 E T5T, T5V, T5S, T5U;
+			 T5T = rio[WS(ios, 11)];
+			 T5V = iio[-WS(ios, 52)];
+			 T5S = W[20];
+			 T5U = W[21];
+			 T5W = FMA(T5S, T5T, T5U * T5V);
+			 T97 = FNMS(T5U, T5T, T5S * T5V);
+		    }
+		    {
+			 E T5Y, T60, T5X, T5Z;
+			 T5Y = rio[WS(ios, 43)];
+			 T60 = iio[-WS(ios, 20)];
+			 T5X = W[84];
+			 T5Z = W[85];
+			 T61 = FMA(T5X, T5Y, T5Z * T60);
+			 T98 = FNMS(T5Z, T5Y, T5X * T60);
+		    }
+		    T62 = T5W + T61;
+		    Te3 = T97 + T98;
+		    T94 = T5W - T61;
+		    T99 = T97 - T98;
+	       }
+	       {
+		    E T5z, T8W, T5E, T8X;
+		    {
+			 E T5w, T5y, T5v, T5x;
+			 T5w = rio[WS(ios, 19)];
+			 T5y = iio[-WS(ios, 44)];
+			 T5v = W[36];
+			 T5x = W[37];
+			 T5z = FMA(T5v, T5w, T5x * T5y);
+			 T8W = FNMS(T5x, T5w, T5v * T5y);
+		    }
+		    {
+			 E T5B, T5D, T5A, T5C;
+			 T5B = rio[WS(ios, 51)];
+			 T5D = iio[-WS(ios, 12)];
+			 T5A = W[100];
+			 T5C = W[101];
+			 T5E = FMA(T5A, T5B, T5C * T5D);
+			 T8X = FNMS(T5C, T5B, T5A * T5D);
+		    }
+		    T5F = T5z + T5E;
+		    TdX = T8W + T8X;
+		    T8T = T5z - T5E;
+		    T8Y = T8W - T8X;
+	       }
+	       {
+		    E T5L, T91, T5Q, T92;
+		    {
+			 E T5I, T5K, T5H, T5J;
+			 T5I = rio[WS(ios, 59)];
+			 T5K = iio[-WS(ios, 4)];
+			 T5H = W[116];
+			 T5J = W[117];
+			 T5L = FMA(T5H, T5I, T5J * T5K);
+			 T91 = FNMS(T5J, T5I, T5H * T5K);
+		    }
+		    {
+			 E T5N, T5P, T5M, T5O;
+			 T5N = rio[WS(ios, 27)];
+			 T5P = iio[-WS(ios, 36)];
+			 T5M = W[52];
+			 T5O = W[53];
+			 T5Q = FMA(T5M, T5N, T5O * T5P);
+			 T92 = FNMS(T5O, T5N, T5M * T5P);
+		    }
+		    T5R = T5L + T5Q;
+		    Te2 = T91 + T92;
+		    T93 = T91 - T92;
+		    T96 = T5L - T5Q;
+	       }
+	       {
+		    E T5G, T63, Te1, Te4;
+		    T5G = T5u + T5F;
+		    T63 = T5R + T62;
+		    T64 = T5G + T63;
+		    TfZ = T63 - T5G;
+		    Te1 = T5R - T62;
+		    Te4 = Te2 - Te3;
+		    Te5 = Te1 + Te4;
+		    Ted = Te1 - Te4;
+	       }
+	       {
+		    E TfS, TfT, T8U, T8Z;
+		    TfS = TdW + TdX;
+		    TfT = Te2 + Te3;
+		    TfU = TfS - TfT;
+		    Tgz = TfS + TfT;
+		    T8U = T8S + T8T;
+		    T8Z = T8V - T8Y;
+		    T90 = FNMS(KP923879532, T8Z, KP382683432 * T8U);
+		    T9o = FMA(KP923879532, T8U, KP382683432 * T8Z);
+	       }
+	       {
+		    E T95, T9a, Tbr, Tbs;
+		    T95 = T93 + T94;
+		    T9a = T96 - T99;
+		    T9b = FMA(KP382683432, T95, KP923879532 * T9a);
+		    T9n = FNMS(KP923879532, T95, KP382683432 * T9a);
+		    Tbr = T93 - T94;
+		    Tbs = T96 + T99;
+		    Tbt = FMA(KP923879532, Tbr, KP382683432 * Tbs);
+		    Tbz = FNMS(KP382683432, Tbr, KP923879532 * Tbs);
+	       }
+	       {
+		    E TdY, TdZ, Tbo, Tbp;
+		    TdY = TdW - TdX;
+		    TdZ = T5u - T5F;
+		    Te0 = TdY - TdZ;
+		    Tee = TdZ + TdY;
+		    Tbo = T8S - T8T;
+		    Tbp = T8V + T8Y;
+		    Tbq = FNMS(KP382683432, Tbp, KP923879532 * Tbo);
+		    TbA = FMA(KP382683432, Tbo, KP923879532 * Tbp);
+	       }
+	  }
+	  {
+	       E T3X, TdB, T8a, T8d, T4v, Tdx, T80, T85, T48, TdC, T8b, T8g, T4k, Tdw, T7X;
+	       E T84;
+	       {
+		    E T3R, T88, T3W, T89;
+		    {
+			 E T3O, T3Q, T3N, T3P;
+			 T3O = rio[WS(ios, 5)];
+			 T3Q = iio[-WS(ios, 58)];
+			 T3N = W[8];
+			 T3P = W[9];
+			 T3R = FMA(T3N, T3O, T3P * T3Q);
+			 T88 = FNMS(T3P, T3O, T3N * T3Q);
+		    }
+		    {
+			 E T3T, T3V, T3S, T3U;
+			 T3T = rio[WS(ios, 37)];
+			 T3V = iio[-WS(ios, 26)];
+			 T3S = W[72];
+			 T3U = W[73];
+			 T3W = FMA(T3S, T3T, T3U * T3V);
+			 T89 = FNMS(T3U, T3T, T3S * T3V);
+		    }
+		    T3X = T3R + T3W;
+		    TdB = T88 + T89;
+		    T8a = T88 - T89;
+		    T8d = T3R - T3W;
+	       }
+	       {
+		    E T4p, T7Y, T4u, T7Z;
+		    {
+			 E T4m, T4o, T4l, T4n;
+			 T4m = rio[WS(ios, 13)];
+			 T4o = iio[-WS(ios, 50)];
+			 T4l = W[24];
+			 T4n = W[25];
+			 T4p = FMA(T4l, T4m, T4n * T4o);
+			 T7Y = FNMS(T4n, T4m, T4l * T4o);
+		    }
+		    {
+			 E T4r, T4t, T4q, T4s;
+			 T4r = rio[WS(ios, 45)];
+			 T4t = iio[-WS(ios, 18)];
+			 T4q = W[88];
+			 T4s = W[89];
+			 T4u = FMA(T4q, T4r, T4s * T4t);
+			 T7Z = FNMS(T4s, T4r, T4q * T4t);
+		    }
+		    T4v = T4p + T4u;
+		    Tdx = T7Y + T7Z;
+		    T80 = T7Y - T7Z;
+		    T85 = T4p - T4u;
+	       }
+	       {
+		    E T42, T8e, T47, T8f;
+		    {
+			 E T3Z, T41, T3Y, T40;
+			 T3Z = rio[WS(ios, 21)];
+			 T41 = iio[-WS(ios, 42)];
+			 T3Y = W[40];
+			 T40 = W[41];
+			 T42 = FMA(T3Y, T3Z, T40 * T41);
+			 T8e = FNMS(T40, T3Z, T3Y * T41);
+		    }
+		    {
+			 E T44, T46, T43, T45;
+			 T44 = rio[WS(ios, 53)];
+			 T46 = iio[-WS(ios, 10)];
+			 T43 = W[104];
+			 T45 = W[105];
+			 T47 = FMA(T43, T44, T45 * T46);
+			 T8f = FNMS(T45, T44, T43 * T46);
+		    }
+		    T48 = T42 + T47;
+		    TdC = T8e + T8f;
+		    T8b = T42 - T47;
+		    T8g = T8e - T8f;
+	       }
+	       {
+		    E T4e, T82, T4j, T83;
+		    {
+			 E T4b, T4d, T4a, T4c;
+			 T4b = rio[WS(ios, 61)];
+			 T4d = iio[-WS(ios, 2)];
+			 T4a = W[120];
+			 T4c = W[121];
+			 T4e = FMA(T4a, T4b, T4c * T4d);
+			 T82 = FNMS(T4c, T4b, T4a * T4d);
+		    }
+		    {
+			 E T4g, T4i, T4f, T4h;
+			 T4g = rio[WS(ios, 29)];
+			 T4i = iio[-WS(ios, 34)];
+			 T4f = W[56];
+			 T4h = W[57];
+			 T4j = FMA(T4f, T4g, T4h * T4i);
+			 T83 = FNMS(T4h, T4g, T4f * T4i);
+		    }
+		    T4k = T4e + T4j;
+		    Tdw = T82 + T83;
+		    T7X = T4e - T4j;
+		    T84 = T82 - T83;
+	       }
+	       {
+		    E T49, T4w, TdA, TdD;
+		    T49 = T3X + T48;
+		    T4w = T4k + T4v;
+		    T4x = T49 + T4w;
+		    TfJ = T4w - T49;
+		    TdA = T3X - T48;
+		    TdD = TdB - TdC;
+		    TdE = TdA + TdD;
+		    TdM = TdD - TdA;
+	       }
+	       {
+		    E TfM, TfN, T81, T86;
+		    TfM = TdB + TdC;
+		    TfN = Tdw + Tdx;
+		    TfO = TfM - TfN;
+		    Tgu = TfM + TfN;
+		    T81 = T7X - T80;
+		    T86 = T84 + T85;
+		    T87 = FNMS(KP923879532, T86, KP382683432 * T81);
+		    T8v = FMA(KP382683432, T86, KP923879532 * T81);
+	       }
+	       {
+		    E T8c, T8h, Tb8, Tb9;
+		    T8c = T8a + T8b;
+		    T8h = T8d - T8g;
+		    T8i = FMA(KP923879532, T8c, KP382683432 * T8h);
+		    T8u = FNMS(KP923879532, T8h, KP382683432 * T8c);
+		    Tb8 = T8a - T8b;
+		    Tb9 = T8d + T8g;
+		    Tba = FMA(KP382683432, Tb8, KP923879532 * Tb9);
+		    Tbg = FNMS(KP382683432, Tb9, KP923879532 * Tb8);
+	       }
+	       {
+		    E Tdv, Tdy, Tb5, Tb6;
+		    Tdv = T4k - T4v;
+		    Tdy = Tdw - Tdx;
+		    Tdz = Tdv - Tdy;
+		    TdN = Tdv + Tdy;
+		    Tb5 = T7X + T80;
+		    Tb6 = T84 - T85;
+		    Tb7 = FNMS(KP382683432, Tb6, KP923879532 * Tb5);
+		    Tbh = FMA(KP923879532, Tb6, KP382683432 * Tb5);
+	       }
+	  }
+	  {
+	       E T1t, Tgn, TgK, TgL, TgV, Th1, T30, Th0, T66, TgX, Tgw, TgE, TgB, TgF, Tgq;
+	       E TgM;
+	       {
+		    E TH, T1s, TgI, TgJ;
+		    TH = Tj + TG;
+		    T1s = T14 + T1r;
+		    T1t = TH + T1s;
+		    Tgn = TH - T1s;
+		    TgI = Tgt + Tgu;
+		    TgJ = Tgy + Tgz;
+		    TgK = TgI - TgJ;
+		    TgL = TgI + TgJ;
+	       }
+	       {
+		    E TgN, TgU, T2e, T2Z;
+		    TgN = Tfq + Tfr;
+		    TgU = TgO + TgT;
+		    TgV = TgN + TgU;
+		    Th1 = TgU - TgN;
+		    T2e = T1Q + T2d;
+		    T2Z = T2B + T2Y;
+		    T30 = T2e + T2Z;
+		    Th0 = T2Z - T2e;
+	       }
+	       {
+		    E T4y, T65, Tgs, Tgv;
+		    T4y = T3M + T4x;
+		    T65 = T5j + T64;
+		    T66 = T4y + T65;
+		    TgX = T65 - T4y;
+		    Tgs = T3M - T4x;
+		    Tgv = Tgt - Tgu;
+		    Tgw = Tgs + Tgv;
+		    TgE = Tgv - Tgs;
+	       }
+	       {
+		    E Tgx, TgA, Tgo, Tgp;
+		    Tgx = T5j - T64;
+		    TgA = Tgy - Tgz;
+		    TgB = Tgx - TgA;
+		    TgF = Tgx + TgA;
+		    Tgo = Tfu + Tfv;
+		    Tgp = TfA + TfB;
+		    Tgq = Tgo - Tgp;
+		    TgM = Tgo + Tgp;
+	       }
+	       {
+		    E T31, TgW, TgH, TgY;
+		    T31 = T1t + T30;
+		    iio[-WS(ios, 32)] = T31 - T66;
+		    rio[0] = T31 + T66;
+		    TgW = TgM + TgV;
+		    rio[WS(ios, 32)] = TgL - TgW;
+		    iio[0] = TgL + TgW;
+		    TgH = T1t - T30;
+		    iio[-WS(ios, 48)] = TgH - TgK;
+		    rio[WS(ios, 16)] = TgH + TgK;
+		    TgY = TgV - TgM;
+		    rio[WS(ios, 48)] = TgX - TgY;
+		    iio[-WS(ios, 16)] = TgX + TgY;
+	       }
+	       {
+		    E Tgr, TgC, TgZ, Th2;
+		    Tgr = Tgn + Tgq;
+		    TgC = KP707106781 * (Tgw + TgB);
+		    iio[-WS(ios, 40)] = Tgr - TgC;
+		    rio[WS(ios, 8)] = Tgr + TgC;
+		    TgZ = KP707106781 * (TgE + TgF);
+		    Th2 = Th0 + Th1;
+		    rio[WS(ios, 40)] = TgZ - Th2;
+		    iio[-WS(ios, 8)] = TgZ + Th2;
+	       }
+	       {
+		    E TgD, TgG, Th3, Th4;
+		    TgD = Tgn - Tgq;
+		    TgG = KP707106781 * (TgE - TgF);
+		    iio[-WS(ios, 56)] = TgD - TgG;
+		    rio[WS(ios, 24)] = TgD + TgG;
+		    Th3 = KP707106781 * (TgB - Tgw);
+		    Th4 = Th1 - Th0;
+		    rio[WS(ios, 56)] = Th3 - Th4;
+		    iio[-WS(ios, 24)] = Th3 + Th4;
+	       }
+	  }
+	  {
+	       E Tft, Tg7, Tgh, Tgl, Th9, Thf, TfE, Th6, TfQ, Tg4, Tga, The, Tge, Tgk, Tg1;
+	       E Tg5;
+	       {
+		    E Tfp, Tfs, Tgf, Tgg;
+		    Tfp = Tj - TG;
+		    Tfs = Tfq - Tfr;
+		    Tft = Tfp - Tfs;
+		    Tg7 = Tfp + Tfs;
+		    Tgf = TfR + TfU;
+		    Tgg = TfY + TfZ;
+		    Tgh = FNMS(KP382683432, Tgg, KP923879532 * Tgf);
+		    Tgl = FMA(KP923879532, Tgg, KP382683432 * Tgf);
+	       }
+	       {
+		    E Th7, Th8, Tfy, TfD;
+		    Th7 = T1r - T14;
+		    Th8 = TgT - TgO;
+		    Th9 = Th7 + Th8;
+		    Thf = Th8 - Th7;
+		    Tfy = Tfw - Tfx;
+		    TfD = Tfz + TfC;
+		    TfE = KP707106781 * (Tfy - TfD);
+		    Th6 = KP707106781 * (Tfy + TfD);
+	       }
+	       {
+		    E TfK, TfP, Tg8, Tg9;
+		    TfK = TfI - TfJ;
+		    TfP = TfL - TfO;
+		    TfQ = FMA(KP923879532, TfK, KP382683432 * TfP);
+		    Tg4 = FNMS(KP923879532, TfP, KP382683432 * TfK);
+		    Tg8 = Tfx + Tfw;
+		    Tg9 = Tfz - TfC;
+		    Tga = KP707106781 * (Tg8 + Tg9);
+		    The = KP707106781 * (Tg9 - Tg8);
+	       }
+	       {
+		    E Tgc, Tgd, TfV, Tg0;
+		    Tgc = TfI + TfJ;
+		    Tgd = TfL + TfO;
+		    Tge = FMA(KP382683432, Tgc, KP923879532 * Tgd);
+		    Tgk = FNMS(KP382683432, Tgd, KP923879532 * Tgc);
+		    TfV = TfR - TfU;
+		    Tg0 = TfY - TfZ;
+		    Tg1 = FNMS(KP923879532, Tg0, KP382683432 * TfV);
+		    Tg5 = FMA(KP382683432, Tg0, KP923879532 * TfV);
+	       }
+	       {
+		    E TfF, Tg2, Thd, Thg;
+		    TfF = Tft + TfE;
+		    Tg2 = TfQ + Tg1;
+		    iio[-WS(ios, 44)] = TfF - Tg2;
+		    rio[WS(ios, 12)] = TfF + Tg2;
+		    Thd = Tg4 + Tg5;
+		    Thg = The + Thf;
+		    rio[WS(ios, 44)] = Thd - Thg;
+		    iio[-WS(ios, 12)] = Thd + Thg;
+	       }
+	       {
+		    E Tg3, Tg6, Thh, Thi;
+		    Tg3 = Tft - TfE;
+		    Tg6 = Tg4 - Tg5;
+		    iio[-WS(ios, 60)] = Tg3 - Tg6;
+		    rio[WS(ios, 28)] = Tg3 + Tg6;
+		    Thh = Tg1 - TfQ;
+		    Thi = Thf - The;
+		    rio[WS(ios, 60)] = Thh - Thi;
+		    iio[-WS(ios, 28)] = Thh + Thi;
+	       }
+	       {
+		    E Tgb, Tgi, Th5, Tha;
+		    Tgb = Tg7 + Tga;
+		    Tgi = Tge + Tgh;
+		    iio[-WS(ios, 36)] = Tgb - Tgi;
+		    rio[WS(ios, 4)] = Tgb + Tgi;
+		    Th5 = Tgk + Tgl;
+		    Tha = Th6 + Th9;
+		    rio[WS(ios, 36)] = Th5 - Tha;
+		    iio[-WS(ios, 4)] = Th5 + Tha;
+	       }
+	       {
+		    E Tgj, Tgm, Thb, Thc;
+		    Tgj = Tg7 - Tga;
+		    Tgm = Tgk - Tgl;
+		    iio[-WS(ios, 52)] = Tgj - Tgm;
+		    rio[WS(ios, 20)] = Tgj + Tgm;
+		    Thb = Tgh - Tge;
+		    Thc = Th9 - Th6;
+		    rio[WS(ios, 52)] = Thb - Thc;
+		    iio[-WS(ios, 20)] = Thb + Thc;
+	       }
+	  }
+	  {
+	       E Td1, Ten, Tdo, ThA, ThD, ThJ, Teq, ThI, Teh, TeB, Tel, Tex, TdQ, TeA, Tek;
+	       E Teu;
+	       {
+		    E TcP, Td0, Teo, Tep;
+		    TcP = TcL - TcO;
+		    Td0 = KP707106781 * (TcU - TcZ);
+		    Td1 = TcP - Td0;
+		    Ten = TcP + Td0;
+		    {
+			 E Tdc, Tdn, ThB, ThC;
+			 Tdc = FNMS(KP923879532, Tdb, KP382683432 * Td6);
+			 Tdn = FMA(KP382683432, Tdh, KP923879532 * Tdm);
+			 Tdo = Tdc - Tdn;
+			 ThA = Tdc + Tdn;
+			 ThB = KP707106781 * (TeF - TeE);
+			 ThC = Thn - Thm;
+			 ThD = ThB + ThC;
+			 ThJ = ThC - ThB;
+		    }
+		    Teo = FMA(KP923879532, Td6, KP382683432 * Tdb);
+		    Tep = FNMS(KP923879532, Tdh, KP382683432 * Tdm);
+		    Teq = Teo + Tep;
+		    ThI = Tep - Teo;
+		    {
+			 E Te7, Tev, Teg, Tew, Te6, Tef;
+			 Te6 = KP707106781 * (Te0 - Te5);
+			 Te7 = TdV - Te6;
+			 Tev = TdV + Te6;
+			 Tef = KP707106781 * (Ted - Tee);
+			 Teg = Tec - Tef;
+			 Tew = Tec + Tef;
+			 Teh = FNMS(KP980785280, Teg, KP195090322 * Te7);
+			 TeB = FMA(KP831469612, Tew, KP555570233 * Tev);
+			 Tel = FMA(KP195090322, Teg, KP980785280 * Te7);
+			 Tex = FNMS(KP555570233, Tew, KP831469612 * Tev);
+		    }
+		    {
+			 E TdG, Tes, TdP, Tet, TdF, TdO;
+			 TdF = KP707106781 * (Tdz - TdE);
+			 TdG = Tdu - TdF;
+			 Tes = Tdu + TdF;
+			 TdO = KP707106781 * (TdM - TdN);
+			 TdP = TdL - TdO;
+			 Tet = TdL + TdO;
+			 TdQ = FMA(KP980785280, TdG, KP195090322 * TdP);
+			 TeA = FNMS(KP555570233, Tet, KP831469612 * Tes);
+			 Tek = FNMS(KP980785280, TdP, KP195090322 * TdG);
+			 Teu = FMA(KP555570233, Tes, KP831469612 * Tet);
+		    }
+	       }
+	       {
+		    E Tdp, Tei, ThH, ThK;
+		    Tdp = Td1 + Tdo;
+		    Tei = TdQ + Teh;
+		    iio[-WS(ios, 46)] = Tdp - Tei;
+		    rio[WS(ios, 14)] = Tdp + Tei;
+		    ThH = Tek + Tel;
+		    ThK = ThI + ThJ;
+		    rio[WS(ios, 46)] = ThH - ThK;
+		    iio[-WS(ios, 14)] = ThH + ThK;
+	       }
+	       {
+		    E Tej, Tem, ThL, ThM;
+		    Tej = Td1 - Tdo;
+		    Tem = Tek - Tel;
+		    iio[-WS(ios, 62)] = Tej - Tem;
+		    rio[WS(ios, 30)] = Tej + Tem;
+		    ThL = Teh - TdQ;
+		    ThM = ThJ - ThI;
+		    rio[WS(ios, 62)] = ThL - ThM;
+		    iio[-WS(ios, 30)] = ThL + ThM;
+	       }
+	       {
+		    E Ter, Tey, Thz, ThE;
+		    Ter = Ten + Teq;
+		    Tey = Teu + Tex;
+		    iio[-WS(ios, 38)] = Ter - Tey;
+		    rio[WS(ios, 6)] = Ter + Tey;
+		    Thz = TeA + TeB;
+		    ThE = ThA + ThD;
+		    rio[WS(ios, 38)] = Thz - ThE;
+		    iio[-WS(ios, 6)] = Thz + ThE;
+	       }
+	       {
+		    E Tez, TeC, ThF, ThG;
+		    Tez = Ten - Teq;
+		    TeC = TeA - TeB;
+		    iio[-WS(ios, 54)] = Tez - TeC;
+		    rio[WS(ios, 22)] = Tez + TeC;
+		    ThF = Tex - Teu;
+		    ThG = ThD - ThA;
+		    rio[WS(ios, 54)] = ThF - ThG;
+		    iio[-WS(ios, 22)] = ThF + ThG;
+	       }
+	  }
+	  {
+	       E TeH, Tf9, TeO, Thk, Thp, Thv, Tfc, Thu, Tf3, Tfn, Tf7, Tfj, TeW, Tfm, Tf6;
+	       E Tfg;
+	       {
+		    E TeD, TeG, Tfa, Tfb;
+		    TeD = TcL + TcO;
+		    TeG = KP707106781 * (TeE + TeF);
+		    TeH = TeD - TeG;
+		    Tf9 = TeD + TeG;
+		    {
+			 E TeK, TeN, Thl, Tho;
+			 TeK = FNMS(KP382683432, TeJ, KP923879532 * TeI);
+			 TeN = FMA(KP923879532, TeL, KP382683432 * TeM);
+			 TeO = TeK - TeN;
+			 Thk = TeK + TeN;
+			 Thl = KP707106781 * (TcU + TcZ);
+			 Tho = Thm + Thn;
+			 Thp = Thl + Tho;
+			 Thv = Tho - Thl;
+		    }
+		    Tfa = FMA(KP382683432, TeI, KP923879532 * TeJ);
+		    Tfb = FNMS(KP382683432, TeL, KP923879532 * TeM);
+		    Tfc = Tfa + Tfb;
+		    Thu = Tfb - Tfa;
+		    {
+			 E TeZ, Tfh, Tf2, Tfi, TeY, Tf1;
+			 TeY = KP707106781 * (Tee + Ted);
+			 TeZ = TeX - TeY;
+			 Tfh = TeX + TeY;
+			 Tf1 = KP707106781 * (Te0 + Te5);
+			 Tf2 = Tf0 - Tf1;
+			 Tfi = Tf0 + Tf1;
+			 Tf3 = FNMS(KP831469612, Tf2, KP555570233 * TeZ);
+			 Tfn = FMA(KP195090322, Tfh, KP980785280 * Tfi);
+			 Tf7 = FMA(KP831469612, TeZ, KP555570233 * Tf2);
+			 Tfj = FNMS(KP195090322, Tfi, KP980785280 * Tfh);
+		    }
+		    {
+			 E TeS, Tfe, TeV, Tff, TeR, TeU;
+			 TeR = KP707106781 * (TdE + Tdz);
+			 TeS = TeQ - TeR;
+			 Tfe = TeQ + TeR;
+			 TeU = KP707106781 * (TdM + TdN);
+			 TeV = TeT - TeU;
+			 Tff = TeT + TeU;
+			 TeW = FMA(KP555570233, TeS, KP831469612 * TeV);
+			 Tfm = FNMS(KP195090322, Tfe, KP980785280 * Tff);
+			 Tf6 = FNMS(KP831469612, TeS, KP555570233 * TeV);
+			 Tfg = FMA(KP980785280, Tfe, KP195090322 * Tff);
+		    }
+	       }
+	       {
+		    E TeP, Tf4, Tht, Thw;
+		    TeP = TeH + TeO;
+		    Tf4 = TeW + Tf3;
+		    iio[-WS(ios, 42)] = TeP - Tf4;
+		    rio[WS(ios, 10)] = TeP + Tf4;
+		    Tht = Tf6 + Tf7;
+		    Thw = Thu + Thv;
+		    rio[WS(ios, 42)] = Tht - Thw;
+		    iio[-WS(ios, 10)] = Tht + Thw;
+	       }
+	       {
+		    E Tf5, Tf8, Thx, Thy;
+		    Tf5 = TeH - TeO;
+		    Tf8 = Tf6 - Tf7;
+		    iio[-WS(ios, 58)] = Tf5 - Tf8;
+		    rio[WS(ios, 26)] = Tf5 + Tf8;
+		    Thx = Tf3 - TeW;
+		    Thy = Thv - Thu;
+		    rio[WS(ios, 58)] = Thx - Thy;
+		    iio[-WS(ios, 26)] = Thx + Thy;
+	       }
+	       {
+		    E Tfd, Tfk, Thj, Thq;
+		    Tfd = Tf9 + Tfc;
+		    Tfk = Tfg + Tfj;
+		    iio[-WS(ios, 34)] = Tfd - Tfk;
+		    rio[WS(ios, 2)] = Tfd + Tfk;
+		    Thj = Tfm + Tfn;
+		    Thq = Thk + Thp;
+		    rio[WS(ios, 34)] = Thj - Thq;
+		    iio[-WS(ios, 2)] = Thj + Thq;
+	       }
+	       {
+		    E Tfl, Tfo, Thr, Ths;
+		    Tfl = Tf9 - Tfc;
+		    Tfo = Tfm - Tfn;
+		    iio[-WS(ios, 50)] = Tfl - Tfo;
+		    rio[WS(ios, 18)] = Tfl + Tfo;
+		    Thr = Tfj - Tfg;
+		    Ths = Thp - Thk;
+		    rio[WS(ios, 50)] = Thr - Ths;
+		    iio[-WS(ios, 18)] = Thr + Ths;
+	       }
+	  }
+	  {
+	       E T6L, T9x, TiD, TiJ, T7E, TiI, T9A, TiA, T8y, T9K, T9u, T9E, T9r, T9L, T9v;
+	       E T9H;
+	       {
+		    E T6n, T6K, TiB, TiC;
+		    T6n = T6b - T6m;
+		    T6K = T6y - T6J;
+		    T6L = T6n - T6K;
+		    T9x = T6n + T6K;
+		    TiB = T9P - T9O;
+		    TiC = Tin - Tim;
+		    TiD = TiB + TiC;
+		    TiJ = TiC - TiB;
+	       }
+	       {
+		    E T7c, T9y, T7D, T9z;
+		    {
+			 E T72, T7b, T7t, T7C;
+			 T72 = T6Q - T71;
+			 T7b = T77 - T7a;
+			 T7c = FNMS(KP980785280, T7b, KP195090322 * T72);
+			 T9y = FMA(KP980785280, T72, KP195090322 * T7b);
+			 T7t = T7h - T7s;
+			 T7C = T7y - T7B;
+			 T7D = FMA(KP195090322, T7t, KP980785280 * T7C);
+			 T9z = FNMS(KP980785280, T7t, KP195090322 * T7C);
+		    }
+		    T7E = T7c - T7D;
+		    TiI = T9z - T9y;
+		    T9A = T9y + T9z;
+		    TiA = T7c + T7D;
+	       }
+	       {
+		    E T8k, T9C, T8x, T9D;
+		    {
+			 E T7W, T8j, T8t, T8w;
+			 T7W = T7K - T7V;
+			 T8j = T87 - T8i;
+			 T8k = T7W - T8j;
+			 T9C = T7W + T8j;
+			 T8t = T8p - T8s;
+			 T8w = T8u - T8v;
+			 T8x = T8t - T8w;
+			 T9D = T8t + T8w;
+		    }
+		    T8y = FMA(KP995184726, T8k, KP098017140 * T8x);
+		    T9K = FNMS(KP634393284, T9D, KP773010453 * T9C);
+		    T9u = FNMS(KP995184726, T8x, KP098017140 * T8k);
+		    T9E = FMA(KP634393284, T9C, KP773010453 * T9D);
+	       }
+	       {
+		    E T9d, T9F, T9q, T9G;
+		    {
+			 E T8P, T9c, T9m, T9p;
+			 T8P = T8D - T8O;
+			 T9c = T90 - T9b;
+			 T9d = T8P - T9c;
+			 T9F = T8P + T9c;
+			 T9m = T9i - T9l;
+			 T9p = T9n - T9o;
+			 T9q = T9m - T9p;
+			 T9G = T9m + T9p;
+		    }
+		    T9r = FNMS(KP995184726, T9q, KP098017140 * T9d);
+		    T9L = FMA(KP773010453, T9G, KP634393284 * T9F);
+		    T9v = FMA(KP098017140, T9q, KP995184726 * T9d);
+		    T9H = FNMS(KP634393284, T9G, KP773010453 * T9F);
+	       }
+	       {
+		    E T7F, T9s, TiH, TiK;
+		    T7F = T6L + T7E;
+		    T9s = T8y + T9r;
+		    iio[-WS(ios, 47)] = T7F - T9s;
+		    rio[WS(ios, 15)] = T7F + T9s;
+		    TiH = T9u + T9v;
+		    TiK = TiI + TiJ;
+		    rio[WS(ios, 47)] = TiH - TiK;
+		    iio[-WS(ios, 15)] = TiH + TiK;
+	       }
+	       {
+		    E T9t, T9w, TiL, TiM;
+		    T9t = T6L - T7E;
+		    T9w = T9u - T9v;
+		    iio[-WS(ios, 63)] = T9t - T9w;
+		    rio[WS(ios, 31)] = T9t + T9w;
+		    TiL = T9r - T8y;
+		    TiM = TiJ - TiI;
+		    rio[WS(ios, 63)] = TiL - TiM;
+		    iio[-WS(ios, 31)] = TiL + TiM;
+	       }
+	       {
+		    E T9B, T9I, Tiz, TiE;
+		    T9B = T9x + T9A;
+		    T9I = T9E + T9H;
+		    iio[-WS(ios, 39)] = T9B - T9I;
+		    rio[WS(ios, 7)] = T9B + T9I;
+		    Tiz = T9K + T9L;
+		    TiE = TiA + TiD;
+		    rio[WS(ios, 39)] = Tiz - TiE;
+		    iio[-WS(ios, 7)] = Tiz + TiE;
+	       }
+	       {
+		    E T9J, T9M, TiF, TiG;
+		    T9J = T9x - T9A;
+		    T9M = T9K - T9L;
+		    iio[-WS(ios, 55)] = T9J - T9M;
+		    rio[WS(ios, 23)] = T9J + T9M;
+		    TiF = T9H - T9E;
+		    TiG = TiD - TiA;
+		    rio[WS(ios, 55)] = TiF - TiG;
+		    iio[-WS(ios, 23)] = TiF + TiG;
+	       }
+	  }
+	  {
+	       E TaL, TbJ, Ti9, Tif, Tb0, Tie, TbM, Ti6, Tbk, TbW, TbG, TbQ, TbD, TbX, TbH;
+	       E TbT;
+	       {
+		    E TaD, TaK, Ti7, Ti8;
+		    TaD = Taz - TaC;
+		    TaK = TaG - TaJ;
+		    TaL = TaD - TaK;
+		    TbJ = TaD + TaK;
+		    Ti7 = Tc1 - Tc0;
+		    Ti8 = ThT - ThQ;
+		    Ti9 = Ti7 + Ti8;
+		    Tif = Ti8 - Ti7;
+	       }
+	       {
+		    E TaS, TbK, TaZ, TbL;
+		    {
+			 E TaO, TaR, TaV, TaY;
+			 TaO = TaM - TaN;
+			 TaR = TaP - TaQ;
+			 TaS = FNMS(KP831469612, TaR, KP555570233 * TaO);
+			 TbK = FMA(KP555570233, TaR, KP831469612 * TaO);
+			 TaV = TaT - TaU;
+			 TaY = TaW - TaX;
+			 TaZ = FMA(KP831469612, TaV, KP555570233 * TaY);
+			 TbL = FNMS(KP831469612, TaY, KP555570233 * TaV);
+		    }
+		    Tb0 = TaS - TaZ;
+		    Tie = TbL - TbK;
+		    TbM = TbK + TbL;
+		    Ti6 = TaS + TaZ;
+	       }
+	       {
+		    E Tbc, TbO, Tbj, TbP;
+		    {
+			 E Tb4, Tbb, Tbf, Tbi;
+			 Tb4 = Tb2 - Tb3;
+			 Tbb = Tb7 - Tba;
+			 Tbc = Tb4 - Tbb;
+			 TbO = Tb4 + Tbb;
+			 Tbf = Tbd - Tbe;
+			 Tbi = Tbg - Tbh;
+			 Tbj = Tbf - Tbi;
+			 TbP = Tbf + Tbi;
+		    }
+		    Tbk = FMA(KP956940335, Tbc, KP290284677 * Tbj);
+		    TbW = FNMS(KP471396736, TbP, KP881921264 * TbO);
+		    TbG = FNMS(KP956940335, Tbj, KP290284677 * Tbc);
+		    TbQ = FMA(KP471396736, TbO, KP881921264 * TbP);
+	       }
+	       {
+		    E Tbv, TbR, TbC, TbS;
+		    {
+			 E Tbn, Tbu, Tby, TbB;
+			 Tbn = Tbl - Tbm;
+			 Tbu = Tbq - Tbt;
+			 Tbv = Tbn - Tbu;
+			 TbR = Tbn + Tbu;
+			 Tby = Tbw - Tbx;
+			 TbB = Tbz - TbA;
+			 TbC = Tby - TbB;
+			 TbS = Tby + TbB;
+		    }
+		    TbD = FNMS(KP956940335, TbC, KP290284677 * Tbv);
+		    TbX = FMA(KP881921264, TbS, KP471396736 * TbR);
+		    TbH = FMA(KP290284677, TbC, KP956940335 * Tbv);
+		    TbT = FNMS(KP471396736, TbS, KP881921264 * TbR);
+	       }
+	       {
+		    E Tb1, TbE, Tid, Tig;
+		    Tb1 = TaL + Tb0;
+		    TbE = Tbk + TbD;
+		    iio[-WS(ios, 45)] = Tb1 - TbE;
+		    rio[WS(ios, 13)] = Tb1 + TbE;
+		    Tid = TbG + TbH;
+		    Tig = Tie + Tif;
+		    rio[WS(ios, 45)] = Tid - Tig;
+		    iio[-WS(ios, 13)] = Tid + Tig;
+	       }
+	       {
+		    E TbF, TbI, Tih, Tii;
+		    TbF = TaL - Tb0;
+		    TbI = TbG - TbH;
+		    iio[-WS(ios, 61)] = TbF - TbI;
+		    rio[WS(ios, 29)] = TbF + TbI;
+		    Tih = TbD - Tbk;
+		    Tii = Tif - Tie;
+		    rio[WS(ios, 61)] = Tih - Tii;
+		    iio[-WS(ios, 29)] = Tih + Tii;
+	       }
+	       {
+		    E TbN, TbU, Ti5, Tia;
+		    TbN = TbJ + TbM;
+		    TbU = TbQ + TbT;
+		    iio[-WS(ios, 37)] = TbN - TbU;
+		    rio[WS(ios, 5)] = TbN + TbU;
+		    Ti5 = TbW + TbX;
+		    Tia = Ti6 + Ti9;
+		    rio[WS(ios, 37)] = Ti5 - Tia;
+		    iio[-WS(ios, 5)] = Ti5 + Tia;
+	       }
+	       {
+		    E TbV, TbY, Tib, Tic;
+		    TbV = TbJ - TbM;
+		    TbY = TbW - TbX;
+		    iio[-WS(ios, 53)] = TbV - TbY;
+		    rio[WS(ios, 21)] = TbV + TbY;
+		    Tib = TbT - TbQ;
+		    Tic = Ti9 - Ti6;
+		    rio[WS(ios, 53)] = Tib - Tic;
+		    iio[-WS(ios, 21)] = Tib + Tic;
+	       }
+	  }
+	  {
+	       E Tc3, Tcv, ThV, Ti1, Tca, Ti0, Tcy, ThO, Tci, TcI, Tcs, TcC, Tcp, TcJ, Tct;
+	       E TcF;
+	       {
+		    E TbZ, Tc2, ThP, ThU;
+		    TbZ = Taz + TaC;
+		    Tc2 = Tc0 + Tc1;
+		    Tc3 = TbZ - Tc2;
+		    Tcv = TbZ + Tc2;
+		    ThP = TaG + TaJ;
+		    ThU = ThQ + ThT;
+		    ThV = ThP + ThU;
+		    Ti1 = ThU - ThP;
+	       }
+	       {
+		    E Tc6, Tcw, Tc9, Tcx;
+		    {
+			 E Tc4, Tc5, Tc7, Tc8;
+			 Tc4 = TaM + TaN;
+			 Tc5 = TaP + TaQ;
+			 Tc6 = FNMS(KP195090322, Tc5, KP980785280 * Tc4);
+			 Tcw = FMA(KP980785280, Tc5, KP195090322 * Tc4);
+			 Tc7 = TaT + TaU;
+			 Tc8 = TaW + TaX;
+			 Tc9 = FMA(KP195090322, Tc7, KP980785280 * Tc8);
+			 Tcx = FNMS(KP195090322, Tc8, KP980785280 * Tc7);
+		    }
+		    Tca = Tc6 - Tc9;
+		    Ti0 = Tcx - Tcw;
+		    Tcy = Tcw + Tcx;
+		    ThO = Tc6 + Tc9;
+	       }
+	       {
+		    E Tce, TcA, Tch, TcB;
+		    {
+			 E Tcc, Tcd, Tcf, Tcg;
+			 Tcc = Tbd + Tbe;
+			 Tcd = Tba + Tb7;
+			 Tce = Tcc - Tcd;
+			 TcA = Tcc + Tcd;
+			 Tcf = Tb2 + Tb3;
+			 Tcg = Tbg + Tbh;
+			 Tch = Tcf - Tcg;
+			 TcB = Tcf + Tcg;
+		    }
+		    Tci = FMA(KP634393284, Tce, KP773010453 * Tch);
+		    TcI = FNMS(KP098017140, TcA, KP995184726 * TcB);
+		    Tcs = FNMS(KP773010453, Tce, KP634393284 * Tch);
+		    TcC = FMA(KP995184726, TcA, KP098017140 * TcB);
+	       }
+	       {
+		    E Tcl, TcD, Tco, TcE;
+		    {
+			 E Tcj, Tck, Tcm, Tcn;
+			 Tcj = Tbl + Tbm;
+			 Tck = TbA + Tbz;
+			 Tcl = Tcj - Tck;
+			 TcD = Tcj + Tck;
+			 Tcm = Tbw + Tbx;
+			 Tcn = Tbq + Tbt;
+			 Tco = Tcm - Tcn;
+			 TcE = Tcm + Tcn;
+		    }
+		    Tcp = FNMS(KP773010453, Tco, KP634393284 * Tcl);
+		    TcJ = FMA(KP098017140, TcD, KP995184726 * TcE);
+		    Tct = FMA(KP773010453, Tcl, KP634393284 * Tco);
+		    TcF = FNMS(KP098017140, TcE, KP995184726 * TcD);
+	       }
+	       {
+		    E Tcb, Tcq, ThZ, Ti2;
+		    Tcb = Tc3 + Tca;
+		    Tcq = Tci + Tcp;
+		    iio[-WS(ios, 41)] = Tcb - Tcq;
+		    rio[WS(ios, 9)] = Tcb + Tcq;
+		    ThZ = Tcs + Tct;
+		    Ti2 = Ti0 + Ti1;
+		    rio[WS(ios, 41)] = ThZ - Ti2;
+		    iio[-WS(ios, 9)] = ThZ + Ti2;
+	       }
+	       {
+		    E Tcr, Tcu, Ti3, Ti4;
+		    Tcr = Tc3 - Tca;
+		    Tcu = Tcs - Tct;
+		    iio[-WS(ios, 57)] = Tcr - Tcu;
+		    rio[WS(ios, 25)] = Tcr + Tcu;
+		    Ti3 = Tcp - Tci;
+		    Ti4 = Ti1 - Ti0;
+		    rio[WS(ios, 57)] = Ti3 - Ti4;
+		    iio[-WS(ios, 25)] = Ti3 + Ti4;
+	       }
+	       {
+		    E Tcz, TcG, ThN, ThW;
+		    Tcz = Tcv + Tcy;
+		    TcG = TcC + TcF;
+		    iio[-WS(ios, 33)] = Tcz - TcG;
+		    rio[WS(ios, 1)] = Tcz + TcG;
+		    ThN = TcI + TcJ;
+		    ThW = ThO + ThV;
+		    rio[WS(ios, 33)] = ThN - ThW;
+		    iio[-WS(ios, 1)] = ThN + ThW;
+	       }
+	       {
+		    E TcH, TcK, ThX, ThY;
+		    TcH = Tcv - Tcy;
+		    TcK = TcI - TcJ;
+		    iio[-WS(ios, 49)] = TcH - TcK;
+		    rio[WS(ios, 17)] = TcH + TcK;
+		    ThX = TcF - TcC;
+		    ThY = ThV - ThO;
+		    rio[WS(ios, 49)] = ThX - ThY;
+		    iio[-WS(ios, 17)] = ThX + ThY;
+	       }
+	  }
+	  {
+	       E T9R, Taj, Tip, Tiv, T9Y, Tiu, Tam, Tik, Ta6, Taw, Tag, Taq, Tad, Tax, Tah;
+	       E Tat;
+	       {
+		    E T9N, T9Q, Til, Tio;
+		    T9N = T6b + T6m;
+		    T9Q = T9O + T9P;
+		    T9R = T9N - T9Q;
+		    Taj = T9N + T9Q;
+		    Til = T6y + T6J;
+		    Tio = Tim + Tin;
+		    Tip = Til + Tio;
+		    Tiv = Tio - Til;
+	       }
+	       {
+		    E T9U, Tak, T9X, Tal;
+		    {
+			 E T9S, T9T, T9V, T9W;
+			 T9S = T6Q + T71;
+			 T9T = T77 + T7a;
+			 T9U = FNMS(KP555570233, T9T, KP831469612 * T9S);
+			 Tak = FMA(KP555570233, T9S, KP831469612 * T9T);
+			 T9V = T7h + T7s;
+			 T9W = T7y + T7B;
+			 T9X = FMA(KP831469612, T9V, KP555570233 * T9W);
+			 Tal = FNMS(KP555570233, T9V, KP831469612 * T9W);
+		    }
+		    T9Y = T9U - T9X;
+		    Tiu = Tal - Tak;
+		    Tam = Tak + Tal;
+		    Tik = T9U + T9X;
+	       }
+	       {
+		    E Ta2, Tao, Ta5, Tap;
+		    {
+			 E Ta0, Ta1, Ta3, Ta4;
+			 Ta0 = T8p + T8s;
+			 Ta1 = T8i + T87;
+			 Ta2 = Ta0 - Ta1;
+			 Tao = Ta0 + Ta1;
+			 Ta3 = T7K + T7V;
+			 Ta4 = T8u + T8v;
+			 Ta5 = Ta3 - Ta4;
+			 Tap = Ta3 + Ta4;
+		    }
+		    Ta6 = FMA(KP471396736, Ta2, KP881921264 * Ta5);
+		    Taw = FNMS(KP290284677, Tao, KP956940335 * Tap);
+		    Tag = FNMS(KP881921264, Ta2, KP471396736 * Ta5);
+		    Taq = FMA(KP956940335, Tao, KP290284677 * Tap);
+	       }
+	       {
+		    E Ta9, Tar, Tac, Tas;
+		    {
+			 E Ta7, Ta8, Taa, Tab;
+			 Ta7 = T8D + T8O;
+			 Ta8 = T9o + T9n;
+			 Ta9 = Ta7 - Ta8;
+			 Tar = Ta7 + Ta8;
+			 Taa = T9i + T9l;
+			 Tab = T90 + T9b;
+			 Tac = Taa - Tab;
+			 Tas = Taa + Tab;
+		    }
+		    Tad = FNMS(KP881921264, Tac, KP471396736 * Ta9);
+		    Tax = FMA(KP290284677, Tar, KP956940335 * Tas);
+		    Tah = FMA(KP881921264, Ta9, KP471396736 * Tac);
+		    Tat = FNMS(KP290284677, Tas, KP956940335 * Tar);
+	       }
+	       {
+		    E T9Z, Tae, Tit, Tiw;
+		    T9Z = T9R + T9Y;
+		    Tae = Ta6 + Tad;
+		    iio[-WS(ios, 43)] = T9Z - Tae;
+		    rio[WS(ios, 11)] = T9Z + Tae;
+		    Tit = Tag + Tah;
+		    Tiw = Tiu + Tiv;
+		    rio[WS(ios, 43)] = Tit - Tiw;
+		    iio[-WS(ios, 11)] = Tit + Tiw;
+	       }
+	       {
+		    E Taf, Tai, Tix, Tiy;
+		    Taf = T9R - T9Y;
+		    Tai = Tag - Tah;
+		    iio[-WS(ios, 59)] = Taf - Tai;
+		    rio[WS(ios, 27)] = Taf + Tai;
+		    Tix = Tad - Ta6;
+		    Tiy = Tiv - Tiu;
+		    rio[WS(ios, 59)] = Tix - Tiy;
+		    iio[-WS(ios, 27)] = Tix + Tiy;
+	       }
+	       {
+		    E Tan, Tau, Tij, Tiq;
+		    Tan = Taj + Tam;
+		    Tau = Taq + Tat;
+		    iio[-WS(ios, 35)] = Tan - Tau;
+		    rio[WS(ios, 3)] = Tan + Tau;
+		    Tij = Taw + Tax;
+		    Tiq = Tik + Tip;
+		    rio[WS(ios, 35)] = Tij - Tiq;
+		    iio[-WS(ios, 3)] = Tij + Tiq;
+	       }
+	       {
+		    E Tav, Tay, Tir, Tis;
+		    Tav = Taj - Tam;
+		    Tay = Taw - Tax;
+		    iio[-WS(ios, 51)] = Tav - Tay;
+		    rio[WS(ios, 19)] = Tav + Tay;
+		    Tir = Tat - Taq;
+		    Tis = Tip - Tik;
+		    rio[WS(ios, 51)] = Tir - Tis;
+		    iio[-WS(ios, 19)] = Tir + Tis;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 64},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 64, "hf_64", twinstr, {808, 270, 230, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_64) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_64, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_7.c b/src/fftw3/rdft/codelets/r2hc/hf_7.c
new file mode 100644
index 0000000..df7b4ac
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_7.c
@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:56 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 7 -dit -name hf_7 -include hf.h */
+
+/*
+ * This function contains 72 FP additions, 60 FP multiplications,
+ * (or, 36 additions, 24 multiplications, 36 fused multiply/add),
+ * 29 stack variables, and 28 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_7.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_7.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_7.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_7(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP222520933, +0.222520933956314404288902564496794759466355569);
+     DK(KP900968867, +0.900968867902419126236102319507445051165919162);
+     DK(KP623489801, +0.623489801858733530525004884004239810632274731);
+     DK(KP433883739, +0.433883739117558120475768332848358754609990728);
+     DK(KP781831482, +0.781831482468029808708444526674057750232334519);
+     DK(KP974927912, +0.974927912181823607018131682993931217232785801);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 12) {
+	  E T1, Tc, TS, TC, TO, TR, Tn, TT, TI, TP, Ty, TU, TF, TQ;
+	  T1 = rio[0];
+	  {
+	       E T6, TA, Tb, TB;
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = rio[WS(ios, 1)];
+		    T5 = iio[-WS(ios, 5)];
+		    T2 = W[0];
+		    T4 = W[1];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    TA = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T8, Ta, T7, T9;
+		    T8 = rio[WS(ios, 6)];
+		    Ta = iio[0];
+		    T7 = W[10];
+		    T9 = W[11];
+		    Tb = FMA(T7, T8, T9 * Ta);
+		    TB = FNMS(T9, T8, T7 * Ta);
+	       }
+	       Tc = T6 + Tb;
+	       TS = Tb - T6;
+	       TC = TA - TB;
+	       TO = TA + TB;
+	  }
+	  TR = iio[-WS(ios, 6)];
+	  {
+	       E Th, TG, Tm, TH;
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = rio[WS(ios, 2)];
+		    Tg = iio[-WS(ios, 4)];
+		    Td = W[2];
+		    Tf = W[3];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    TG = FNMS(Tf, Te, Td * Tg);
+	       }
+	       {
+		    E Tj, Tl, Ti, Tk;
+		    Tj = rio[WS(ios, 5)];
+		    Tl = iio[-WS(ios, 1)];
+		    Ti = W[8];
+		    Tk = W[9];
+		    Tm = FMA(Ti, Tj, Tk * Tl);
+		    TH = FNMS(Tk, Tj, Ti * Tl);
+	       }
+	       Tn = Th + Tm;
+	       TT = Tm - Th;
+	       TI = TG - TH;
+	       TP = TG + TH;
+	  }
+	  {
+	       E Ts, TD, Tx, TE;
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = rio[WS(ios, 3)];
+		    Tr = iio[-WS(ios, 3)];
+		    To = W[4];
+		    Tq = W[5];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    TD = FNMS(Tq, Tp, To * Tr);
+	       }
+	       {
+		    E Tu, Tw, Tt, Tv;
+		    Tu = rio[WS(ios, 4)];
+		    Tw = iio[-WS(ios, 2)];
+		    Tt = W[6];
+		    Tv = W[7];
+		    Tx = FMA(Tt, Tu, Tv * Tw);
+		    TE = FNMS(Tv, Tu, Tt * Tw);
+	       }
+	       Ty = Ts + Tx;
+	       TU = Tx - Ts;
+	       TF = TD - TE;
+	       TQ = TD + TE;
+	  }
+	  rio[0] = T1 + Tc + Tn + Ty;
+	  iio[0] = TO + TP + TQ + TR;
+	  {
+	       E TJ, Tz, TX, TY;
+	       TJ = FNMS(KP781831482, TF, KP974927912 * TC) - (KP433883739 * TI);
+	       Tz = FMA(KP623489801, Ty, T1) + FNMA(KP900968867, Tn, KP222520933 * Tc);
+	       iio[-WS(ios, 5)] = Tz - TJ;
+	       rio[WS(ios, 2)] = Tz + TJ;
+	       TX = FNMS(KP781831482, TU, KP974927912 * TS) - (KP433883739 * TT);
+	       TY = FMA(KP623489801, TQ, TR) + FNMA(KP900968867, TP, KP222520933 * TO);
+	       rio[WS(ios, 5)] = TX - TY;
+	       iio[-WS(ios, 2)] = TX + TY;
+	  }
+	  {
+	       E TL, TK, TV, TW;
+	       TL = FMA(KP781831482, TC, KP974927912 * TI) + (KP433883739 * TF);
+	       TK = FMA(KP623489801, Tc, T1) + FNMA(KP900968867, Ty, KP222520933 * Tn);
+	       iio[-WS(ios, 6)] = TK - TL;
+	       rio[WS(ios, 1)] = TK + TL;
+	       TV = FMA(KP781831482, TS, KP974927912 * TT) + (KP433883739 * TU);
+	       TW = FMA(KP623489801, TO, TR) + FNMA(KP900968867, TQ, KP222520933 * TP);
+	       rio[WS(ios, 6)] = TV - TW;
+	       iio[-WS(ios, 1)] = TV + TW;
+	  }
+	  {
+	       E TN, TM, TZ, T10;
+	       TN = FMA(KP433883739, TC, KP974927912 * TF) - (KP781831482 * TI);
+	       TM = FMA(KP623489801, Tn, T1) + FNMA(KP222520933, Ty, KP900968867 * Tc);
+	       iio[-WS(ios, 4)] = TM - TN;
+	       rio[WS(ios, 3)] = TM + TN;
+	       TZ = FMA(KP433883739, TS, KP974927912 * TU) - (KP781831482 * TT);
+	       T10 = FMA(KP623489801, TP, TR) + FNMA(KP222520933, TQ, KP900968867 * TO);
+	       rio[WS(ios, 4)] = TZ - T10;
+	       iio[-WS(ios, 3)] = TZ + T10;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 7},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 7, "hf_7", twinstr, {36, 24, 36, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_7) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_7, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_8.c b/src/fftw3/rdft/codelets/r2hc/hf_8.c
new file mode 100644
index 0000000..21dedce
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_8.c
@@ -0,0 +1,203 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:58 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 8 -dit -name hf_8 -include hf.h */
+
+/*
+ * This function contains 66 FP additions, 32 FP multiplications,
+ * (or, 52 additions, 18 multiplications, 14 fused multiply/add),
+ * 28 stack variables, and 32 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_8(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 14) {
+	  E T7, T1e, TH, T19, TF, T13, TR, TU, Ti, T1f, TK, T16, Tu, T12, TM;
+	  E TP;
+	  {
+	       E T1, T18, T6, T17;
+	       T1 = rio[0];
+	       T18 = iio[-WS(ios, 7)];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = rio[WS(ios, 4)];
+		    T5 = iio[-WS(ios, 3)];
+		    T2 = W[6];
+		    T4 = W[7];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    T17 = FNMS(T4, T3, T2 * T5);
+	       }
+	       T7 = T1 + T6;
+	       T1e = T18 - T17;
+	       TH = T1 - T6;
+	       T19 = T17 + T18;
+	  }
+	  {
+	       E Tz, TS, TE, TT;
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = rio[WS(ios, 7)];
+		    Ty = iio[0];
+		    Tv = W[12];
+		    Tx = W[13];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    TS = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = rio[WS(ios, 3)];
+		    TD = iio[-WS(ios, 4)];
+		    TA = W[4];
+		    TC = W[5];
+		    TE = FMA(TA, TB, TC * TD);
+		    TT = FNMS(TC, TB, TA * TD);
+	       }
+	       TF = Tz + TE;
+	       T13 = TS + TT;
+	       TR = Tz - TE;
+	       TU = TS - TT;
+	  }
+	  {
+	       E Tc, TI, Th, TJ;
+	       {
+		    E T9, Tb, T8, Ta;
+		    T9 = rio[WS(ios, 2)];
+		    Tb = iio[-WS(ios, 5)];
+		    T8 = W[2];
+		    Ta = W[3];
+		    Tc = FMA(T8, T9, Ta * Tb);
+		    TI = FNMS(Ta, T9, T8 * Tb);
+	       }
+	       {
+		    E Te, Tg, Td, Tf;
+		    Te = rio[WS(ios, 6)];
+		    Tg = iio[-WS(ios, 1)];
+		    Td = W[10];
+		    Tf = W[11];
+		    Th = FMA(Td, Te, Tf * Tg);
+		    TJ = FNMS(Tf, Te, Td * Tg);
+	       }
+	       Ti = Tc + Th;
+	       T1f = Tc - Th;
+	       TK = TI - TJ;
+	       T16 = TI + TJ;
+	  }
+	  {
+	       E To, TN, Tt, TO;
+	       {
+		    E Tl, Tn, Tk, Tm;
+		    Tl = rio[WS(ios, 1)];
+		    Tn = iio[-WS(ios, 6)];
+		    Tk = W[0];
+		    Tm = W[1];
+		    To = FMA(Tk, Tl, Tm * Tn);
+		    TN = FNMS(Tm, Tl, Tk * Tn);
+	       }
+	       {
+		    E Tq, Ts, Tp, Tr;
+		    Tq = rio[WS(ios, 5)];
+		    Ts = iio[-WS(ios, 2)];
+		    Tp = W[8];
+		    Tr = W[9];
+		    Tt = FMA(Tp, Tq, Tr * Ts);
+		    TO = FNMS(Tr, Tq, Tp * Ts);
+	       }
+	       Tu = To + Tt;
+	       T12 = TN + TO;
+	       TM = To - Tt;
+	       TP = TN - TO;
+	  }
+	  {
+	       E Tj, TG, T1b, T1c;
+	       Tj = T7 + Ti;
+	       TG = Tu + TF;
+	       iio[-WS(ios, 4)] = Tj - TG;
+	       rio[0] = Tj + TG;
+	       {
+		    E T15, T1a, T11, T14;
+		    T15 = T12 + T13;
+		    T1a = T16 + T19;
+		    rio[WS(ios, 4)] = T15 - T1a;
+		    iio[0] = T15 + T1a;
+		    T11 = T7 - Ti;
+		    T14 = T12 - T13;
+		    iio[-WS(ios, 6)] = T11 - T14;
+		    rio[WS(ios, 2)] = T11 + T14;
+	       }
+	       T1b = TF - Tu;
+	       T1c = T19 - T16;
+	       rio[WS(ios, 6)] = T1b - T1c;
+	       iio[-WS(ios, 2)] = T1b + T1c;
+	       {
+		    E TX, T1g, T10, T1d, TY, TZ;
+		    TX = TH - TK;
+		    T1g = T1e - T1f;
+		    TY = TP - TM;
+		    TZ = TR + TU;
+		    T10 = KP707106781 * (TY - TZ);
+		    T1d = KP707106781 * (TY + TZ);
+		    iio[-WS(ios, 7)] = TX - T10;
+		    iio[-WS(ios, 1)] = T1d + T1g;
+		    rio[WS(ios, 3)] = TX + T10;
+		    rio[WS(ios, 5)] = T1d - T1g;
+	       }
+	       {
+		    E TL, T1i, TW, T1h, TQ, TV;
+		    TL = TH + TK;
+		    T1i = T1f + T1e;
+		    TQ = TM + TP;
+		    TV = TR - TU;
+		    TW = KP707106781 * (TQ + TV);
+		    T1h = KP707106781 * (TV - TQ);
+		    iio[-WS(ios, 5)] = TL - TW;
+		    iio[-WS(ios, 3)] = T1h + T1i;
+		    rio[WS(ios, 1)] = TL + TW;
+		    rio[WS(ios, 7)] = T1h - T1i;
+	       }
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 8},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 8, "hf_8", twinstr, {52, 18, 14, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_8) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_8, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/hf_9.c b/src/fftw3/rdft/codelets/r2hc/hf_9.c
new file mode 100644
index 0000000..167bd52
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/hf_9.c
@@ -0,0 +1,256 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:57:01 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_hc2hc -compact -variables 4 -n 9 -dit -name hf_9 -include hf.h */
+
+/*
+ * This function contains 96 FP additions, 72 FP multiplications,
+ * (or, 60 additions, 36 multiplications, 36 fused multiply/add),
+ * 41 stack variables, and 36 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: hf_9.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_9.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: hf_9.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "hf.h"
+
+static const R *hf_9(R *rio, R *iio, const R *W, stride ios, int m, int dist)
+{
+     DK(KP939692620, +0.939692620785908384054109277324731469936208134);
+     DK(KP342020143, +0.342020143325668733044099614682259580763083368);
+     DK(KP984807753, +0.984807753012208059366743024589523013670643252);
+     DK(KP173648177, +0.173648177666930348851716626769314796000375677);
+     DK(KP642787609, +0.642787609686539326322643409907263432907559884);
+     DK(KP766044443, +0.766044443118978035202392650555416673935832457);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = m - 2; i > 0; i = i - 2, rio = rio + dist, iio = iio - dist, W = W + 16) {
+	  E T1, T1B, TQ, T1G, Tc, TN, T1A, T1H, TL, T1x, T17, T1o, T1c, T1n, Tu;
+	  E T1w, TW, T1k, T11, T1l;
+	  {
+	       E T6, TO, Tb, TP;
+	       T1 = rio[0];
+	       T1B = iio[-WS(ios, 8)];
+	       {
+		    E T3, T5, T2, T4;
+		    T3 = rio[WS(ios, 3)];
+		    T5 = iio[-WS(ios, 5)];
+		    T2 = W[4];
+		    T4 = W[5];
+		    T6 = FMA(T2, T3, T4 * T5);
+		    TO = FNMS(T4, T3, T2 * T5);
+	       }
+	       {
+		    E T8, Ta, T7, T9;
+		    T8 = rio[WS(ios, 6)];
+		    Ta = iio[-WS(ios, 2)];
+		    T7 = W[10];
+		    T9 = W[11];
+		    Tb = FMA(T7, T8, T9 * Ta);
+		    TP = FNMS(T9, T8, T7 * Ta);
+	       }
+	       TQ = KP866025403 * (TO - TP);
+	       T1G = KP866025403 * (Tb - T6);
+	       Tc = T6 + Tb;
+	       TN = FNMS(KP500000000, Tc, T1);
+	       T1A = TO + TP;
+	       T1H = FNMS(KP500000000, T1A, T1B);
+	  }
+	  {
+	       E Tz, T19, TE, T14, TJ, T15, TK, T1a;
+	       {
+		    E Tw, Ty, Tv, Tx;
+		    Tw = rio[WS(ios, 2)];
+		    Ty = iio[-WS(ios, 6)];
+		    Tv = W[2];
+		    Tx = W[3];
+		    Tz = FMA(Tv, Tw, Tx * Ty);
+		    T19 = FNMS(Tx, Tw, Tv * Ty);
+	       }
+	       {
+		    E TB, TD, TA, TC;
+		    TB = rio[WS(ios, 5)];
+		    TD = iio[-WS(ios, 3)];
+		    TA = W[8];
+		    TC = W[9];
+		    TE = FMA(TA, TB, TC * TD);
+		    T14 = FNMS(TC, TB, TA * TD);
+	       }
+	       {
+		    E TG, TI, TF, TH;
+		    TG = rio[WS(ios, 8)];
+		    TI = iio[0];
+		    TF = W[14];
+		    TH = W[15];
+		    TJ = FMA(TF, TG, TH * TI);
+		    T15 = FNMS(TH, TG, TF * TI);
+	       }
+	       TK = TE + TJ;
+	       T1a = T14 + T15;
+	       TL = Tz + TK;
+	       T1x = T19 + T1a;
+	       {
+		    E T13, T16, T18, T1b;
+		    T13 = FNMS(KP500000000, TK, Tz);
+		    T16 = KP866025403 * (T14 - T15);
+		    T17 = T13 + T16;
+		    T1o = T13 - T16;
+		    T18 = KP866025403 * (TJ - TE);
+		    T1b = FNMS(KP500000000, T1a, T19);
+		    T1c = T18 + T1b;
+		    T1n = T1b - T18;
+	       }
+	  }
+	  {
+	       E Ti, TY, Tn, TT, Ts, TU, Tt, TZ;
+	       {
+		    E Tf, Th, Te, Tg;
+		    Tf = rio[WS(ios, 1)];
+		    Th = iio[-WS(ios, 7)];
+		    Te = W[0];
+		    Tg = W[1];
+		    Ti = FMA(Te, Tf, Tg * Th);
+		    TY = FNMS(Tg, Tf, Te * Th);
+	       }
+	       {
+		    E Tk, Tm, Tj, Tl;
+		    Tk = rio[WS(ios, 4)];
+		    Tm = iio[-WS(ios, 4)];
+		    Tj = W[6];
+		    Tl = W[7];
+		    Tn = FMA(Tj, Tk, Tl * Tm);
+		    TT = FNMS(Tl, Tk, Tj * Tm);
+	       }
+	       {
+		    E Tp, Tr, To, Tq;
+		    Tp = rio[WS(ios, 7)];
+		    Tr = iio[-WS(ios, 1)];
+		    To = W[12];
+		    Tq = W[13];
+		    Ts = FMA(To, Tp, Tq * Tr);
+		    TU = FNMS(Tq, Tp, To * Tr);
+	       }
+	       Tt = Tn + Ts;
+	       TZ = TT + TU;
+	       Tu = Ti + Tt;
+	       T1w = TY + TZ;
+	       {
+		    E TS, TV, TX, T10;
+		    TS = FNMS(KP500000000, Tt, Ti);
+		    TV = KP866025403 * (TT - TU);
+		    TW = TS + TV;
+		    T1k = TS - TV;
+		    TX = KP866025403 * (Ts - Tn);
+		    T10 = FNMS(KP500000000, TZ, TY);
+		    T11 = TX + T10;
+		    T1l = T10 - TX;
+	       }
+	  }
+	  {
+	       E T1y, Td, TM, T1v;
+	       T1y = KP866025403 * (T1w - T1x);
+	       Td = T1 + Tc;
+	       TM = Tu + TL;
+	       T1v = FNMS(KP500000000, TM, Td);
+	       rio[0] = Td + TM;
+	       rio[WS(ios, 3)] = T1v + T1y;
+	       iio[-WS(ios, 6)] = T1v - T1y;
+	  }
+	  {
+	       E T1D, T1z, T1C, T1E;
+	       T1D = KP866025403 * (TL - Tu);
+	       T1z = T1w + T1x;
+	       T1C = T1A + T1B;
+	       T1E = FNMS(KP500000000, T1z, T1C);
+	       iio[0] = T1z + T1C;
+	       iio[-WS(ios, 3)] = T1D + T1E;
+	       rio[WS(ios, 6)] = T1D - T1E;
+	  }
+	  {
+	       E TR, T1I, T1e, T1J, T1i, T1F, T1f, T1K;
+	       TR = TN + TQ;
+	       T1I = T1G + T1H;
+	       {
+		    E T12, T1d, T1g, T1h;
+		    T12 = FMA(KP766044443, TW, KP642787609 * T11);
+		    T1d = FMA(KP173648177, T17, KP984807753 * T1c);
+		    T1e = T12 + T1d;
+		    T1J = KP866025403 * (T1d - T12);
+		    T1g = FNMS(KP642787609, TW, KP766044443 * T11);
+		    T1h = FNMS(KP984807753, T17, KP173648177 * T1c);
+		    T1i = KP866025403 * (T1g - T1h);
+		    T1F = T1g + T1h;
+	       }
+	       rio[WS(ios, 1)] = TR + T1e;
+	       iio[-WS(ios, 1)] = T1F + T1I;
+	       T1f = FNMS(KP500000000, T1e, TR);
+	       iio[-WS(ios, 7)] = T1f - T1i;
+	       rio[WS(ios, 4)] = T1f + T1i;
+	       T1K = FNMS(KP500000000, T1F, T1I);
+	       rio[WS(ios, 7)] = T1J - T1K;
+	       iio[-WS(ios, 4)] = T1J + T1K;
+	  }
+	  {
+	       E T1j, T1M, T1q, T1O, T1u, T1L, T1r, T1N;
+	       T1j = TN - TQ;
+	       T1M = T1H - T1G;
+	       {
+		    E T1m, T1p, T1s, T1t;
+		    T1m = FMA(KP173648177, T1k, KP984807753 * T1l);
+		    T1p = FNMS(KP939692620, T1o, KP342020143 * T1n);
+		    T1q = T1m + T1p;
+		    T1O = KP866025403 * (T1p - T1m);
+		    T1s = FNMS(KP984807753, T1k, KP173648177 * T1l);
+		    T1t = FMA(KP342020143, T1o, KP939692620 * T1n);
+		    T1u = KP866025403 * (T1s + T1t);
+		    T1L = T1s - T1t;
+	       }
+	       rio[WS(ios, 2)] = T1j + T1q;
+	       iio[-WS(ios, 2)] = T1L + T1M;
+	       T1r = FNMS(KP500000000, T1q, T1j);
+	       iio[-WS(ios, 8)] = T1r - T1u;
+	       iio[-WS(ios, 5)] = T1r + T1u;
+	       T1N = FMS(KP500000000, T1L, T1M);
+	       rio[WS(ios, 5)] = T1N - T1O;
+	       rio[WS(ios, 8)] = T1O + T1N;
+	  }
+     }
+     return W;
+}
+
+static const tw_instr twinstr[] = {
+     {TW_FULL, 0, 9},
+     {TW_NEXT, 1, 0}
+};
+
+static const hc2hc_desc desc = { 9, "hf_9", twinstr, {60, 36, 36, 0}, &GENUS, 0, 0, 0 };
+
+void X(codelet_hf_9) (planner *p) {
+     X(khc2hc_dit_register) (p, hf_9, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/mr2hcII_32.c b/src/fftw3/rdft/codelets/r2hc/mr2hcII_32.c
new file mode 100644
index 0000000..38c2613
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/mr2hcII_32.c
@@ -0,0 +1,368 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:43 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc_noinline -compact -variables 4 -n 32 -name mr2hcII_32 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 174 FP additions, 82 FP multiplications,
+ * (or, 138 additions, 46 multiplications, 36 fused multiply/add),
+ * 61 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: mr2hcII_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: mr2hcII_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: mr2hcII_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void mr2hcII_32_0(const R *I, R *ro, R *io, stride is, stride ros, stride ios)
+{
+     DK(KP471396736, +0.471396736825997648556387625905254377657460319);
+     DK(KP881921264, +0.881921264348355029712756863660388349508442621);
+     DK(KP634393284, +0.634393284163645498215171613225493370675687095);
+     DK(KP773010453, +0.773010453362736960810906609758469800971041293);
+     DK(KP290284677, +0.290284677254462367636192375817395274691476278);
+     DK(KP956940335, +0.956940335732208864935797886980269969482849206);
+     DK(KP995184726, +0.995184726672196886244836953109479921575474869);
+     DK(KP098017140, +0.098017140329560601994195563888641845861136673);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     {
+	  E T5, T2D, T1z, T2q, Tc, T2C, T1C, T2n, Tm, T1k, T1J, T26, Tv, T1l, T1G;
+	  E T27, T15, T1r, T1Y, T2e, T1c, T1s, T1V, T2d, TK, T1o, T1R, T2b, TR, T1p;
+	  E T1O, T2a;
+	  {
+	       E T1, T2p, T4, T2o, T2, T3;
+	       T1 = I[0];
+	       T2p = I[WS(is, 16)];
+	       T2 = I[WS(is, 8)];
+	       T3 = I[WS(is, 24)];
+	       T4 = KP707106781 * (T2 - T3);
+	       T2o = KP707106781 * (T2 + T3);
+	       T5 = T1 + T4;
+	       T2D = T2p - T2o;
+	       T1z = T1 - T4;
+	       T2q = T2o + T2p;
+	  }
+	  {
+	       E T8, T1A, Tb, T1B;
+	       {
+		    E T6, T7, T9, Ta;
+		    T6 = I[WS(is, 4)];
+		    T7 = I[WS(is, 20)];
+		    T8 = FNMS(KP382683432, T7, KP923879532 * T6);
+		    T1A = FMA(KP382683432, T6, KP923879532 * T7);
+		    T9 = I[WS(is, 12)];
+		    Ta = I[WS(is, 28)];
+		    Tb = FNMS(KP923879532, Ta, KP382683432 * T9);
+		    T1B = FMA(KP923879532, T9, KP382683432 * Ta);
+	       }
+	       Tc = T8 + Tb;
+	       T2C = Tb - T8;
+	       T1C = T1A - T1B;
+	       T2n = T1A + T1B;
+	  }
+	  {
+	       E Te, Tk, Th, Tj, Tf, Tg;
+	       Te = I[WS(is, 2)];
+	       Tk = I[WS(is, 18)];
+	       Tf = I[WS(is, 10)];
+	       Tg = I[WS(is, 26)];
+	       Th = KP707106781 * (Tf - Tg);
+	       Tj = KP707106781 * (Tf + Tg);
+	       {
+		    E Ti, Tl, T1H, T1I;
+		    Ti = Te + Th;
+		    Tl = Tj + Tk;
+		    Tm = FNMS(KP195090322, Tl, KP980785280 * Ti);
+		    T1k = FMA(KP195090322, Ti, KP980785280 * Tl);
+		    T1H = Tk - Tj;
+		    T1I = Te - Th;
+		    T1J = FNMS(KP555570233, T1I, KP831469612 * T1H);
+		    T26 = FMA(KP831469612, T1I, KP555570233 * T1H);
+	       }
+	  }
+	  {
+	       E Tq, Tt, Tp, Ts, Tn, To;
+	       Tq = I[WS(is, 30)];
+	       Tt = I[WS(is, 14)];
+	       Tn = I[WS(is, 6)];
+	       To = I[WS(is, 22)];
+	       Tp = KP707106781 * (Tn - To);
+	       Ts = KP707106781 * (Tn + To);
+	       {
+		    E Tr, Tu, T1E, T1F;
+		    Tr = Tp - Tq;
+		    Tu = Ts + Tt;
+		    Tv = FMA(KP980785280, Tr, KP195090322 * Tu);
+		    T1l = FNMS(KP980785280, Tu, KP195090322 * Tr);
+		    T1E = Tt - Ts;
+		    T1F = Tp + Tq;
+		    T1G = FNMS(KP555570233, T1F, KP831469612 * T1E);
+		    T27 = FMA(KP831469612, T1F, KP555570233 * T1E);
+	       }
+	  }
+	  {
+	       E TW, T1a, TV, T19, T10, T16, T13, T17, TT, TU;
+	       TW = I[WS(is, 31)];
+	       T1a = I[WS(is, 15)];
+	       TT = I[WS(is, 7)];
+	       TU = I[WS(is, 23)];
+	       TV = KP707106781 * (TT - TU);
+	       T19 = KP707106781 * (TT + TU);
+	       {
+		    E TY, TZ, T11, T12;
+		    TY = I[WS(is, 3)];
+		    TZ = I[WS(is, 19)];
+		    T10 = FNMS(KP382683432, TZ, KP923879532 * TY);
+		    T16 = FMA(KP382683432, TY, KP923879532 * TZ);
+		    T11 = I[WS(is, 11)];
+		    T12 = I[WS(is, 27)];
+		    T13 = FNMS(KP923879532, T12, KP382683432 * T11);
+		    T17 = FMA(KP923879532, T11, KP382683432 * T12);
+	       }
+	       {
+		    E TX, T14, T1W, T1X;
+		    TX = TV - TW;
+		    T14 = T10 + T13;
+		    T15 = TX + T14;
+		    T1r = TX - T14;
+		    T1W = T13 - T10;
+		    T1X = T1a - T19;
+		    T1Y = T1W - T1X;
+		    T2e = T1W + T1X;
+	       }
+	       {
+		    E T18, T1b, T1T, T1U;
+		    T18 = T16 + T17;
+		    T1b = T19 + T1a;
+		    T1c = T18 + T1b;
+		    T1s = T1b - T18;
+		    T1T = TV + TW;
+		    T1U = T16 - T17;
+		    T1V = T1T + T1U;
+		    T2d = T1U - T1T;
+	       }
+	  }
+	  {
+	       E Ty, TP, TB, TO, TF, TL, TI, TM, Tz, TA;
+	       Ty = I[WS(is, 1)];
+	       TP = I[WS(is, 17)];
+	       Tz = I[WS(is, 9)];
+	       TA = I[WS(is, 25)];
+	       TB = KP707106781 * (Tz - TA);
+	       TO = KP707106781 * (Tz + TA);
+	       {
+		    E TD, TE, TG, TH;
+		    TD = I[WS(is, 5)];
+		    TE = I[WS(is, 21)];
+		    TF = FNMS(KP382683432, TE, KP923879532 * TD);
+		    TL = FMA(KP382683432, TD, KP923879532 * TE);
+		    TG = I[WS(is, 13)];
+		    TH = I[WS(is, 29)];
+		    TI = FNMS(KP923879532, TH, KP382683432 * TG);
+		    TM = FMA(KP923879532, TG, KP382683432 * TH);
+	       }
+	       {
+		    E TC, TJ, T1P, T1Q;
+		    TC = Ty + TB;
+		    TJ = TF + TI;
+		    TK = TC + TJ;
+		    T1o = TC - TJ;
+		    T1P = TI - TF;
+		    T1Q = TP - TO;
+		    T1R = T1P - T1Q;
+		    T2b = T1P + T1Q;
+	       }
+	       {
+		    E TN, TQ, T1M, T1N;
+		    TN = TL + TM;
+		    TQ = TO + TP;
+		    TR = TN + TQ;
+		    T1p = TQ - TN;
+		    T1M = Ty - TB;
+		    T1N = TL - TM;
+		    T1O = T1M - T1N;
+		    T2a = T1M + T1N;
+	       }
+	  }
+	  {
+	       E Tx, T1f, T2s, T2u, T1e, T2l, T1i, T2t;
+	       {
+		    E Td, Tw, T2m, T2r;
+		    Td = T5 + Tc;
+		    Tw = Tm + Tv;
+		    Tx = Td - Tw;
+		    T1f = Td + Tw;
+		    T2m = T1l - T1k;
+		    T2r = T2n + T2q;
+		    T2s = T2m - T2r;
+		    T2u = T2m + T2r;
+	       }
+	       {
+		    E TS, T1d, T1g, T1h;
+		    TS = FMA(KP098017140, TK, KP995184726 * TR);
+		    T1d = FNMS(KP995184726, T1c, KP098017140 * T15);
+		    T1e = TS + T1d;
+		    T2l = T1d - TS;
+		    T1g = FNMS(KP098017140, TR, KP995184726 * TK);
+		    T1h = FMA(KP995184726, T15, KP098017140 * T1c);
+		    T1i = T1g + T1h;
+		    T2t = T1h - T1g;
+	       }
+	       ro[WS(ros, 8)] = Tx - T1e;
+	       io[WS(ios, 8)] = T2t - T2u;
+	       ro[WS(ros, 7)] = Tx + T1e;
+	       io[WS(ios, 7)] = T2t + T2u;
+	       ro[WS(ros, 15)] = T1f - T1i;
+	       io[WS(ios, 15)] = T2l - T2s;
+	       ro[0] = T1f + T1i;
+	       io[0] = T2l + T2s;
+	  }
+	  {
+	       E T29, T2h, T2M, T2O, T2g, T2J, T2k, T2N;
+	       {
+		    E T25, T28, T2K, T2L;
+		    T25 = T1z + T1C;
+		    T28 = T26 - T27;
+		    T29 = T25 + T28;
+		    T2h = T25 - T28;
+		    T2K = T1J + T1G;
+		    T2L = T2C + T2D;
+		    T2M = T2K - T2L;
+		    T2O = T2K + T2L;
+	       }
+	       {
+		    E T2c, T2f, T2i, T2j;
+		    T2c = FMA(KP956940335, T2a, KP290284677 * T2b);
+		    T2f = FNMS(KP290284677, T2e, KP956940335 * T2d);
+		    T2g = T2c + T2f;
+		    T2J = T2f - T2c;
+		    T2i = FMA(KP290284677, T2d, KP956940335 * T2e);
+		    T2j = FNMS(KP290284677, T2a, KP956940335 * T2b);
+		    T2k = T2i - T2j;
+		    T2N = T2j + T2i;
+	       }
+	       ro[WS(ros, 14)] = T29 - T2g;
+	       io[WS(ios, 14)] = T2N - T2O;
+	       ro[WS(ros, 1)] = T29 + T2g;
+	       io[WS(ios, 1)] = T2N + T2O;
+	       ro[WS(ros, 9)] = T2h - T2k;
+	       io[WS(ios, 9)] = T2J - T2M;
+	       ro[WS(ros, 6)] = T2h + T2k;
+	       io[WS(ios, 6)] = T2J + T2M;
+	  }
+	  {
+	       E T1n, T1v, T2y, T2A, T1u, T2v, T1y, T2z;
+	       {
+		    E T1j, T1m, T2w, T2x;
+		    T1j = T5 - Tc;
+		    T1m = T1k + T1l;
+		    T1n = T1j + T1m;
+		    T1v = T1j - T1m;
+		    T2w = Tv - Tm;
+		    T2x = T2q - T2n;
+		    T2y = T2w - T2x;
+		    T2A = T2w + T2x;
+	       }
+	       {
+		    E T1q, T1t, T1w, T1x;
+		    T1q = FMA(KP773010453, T1o, KP634393284 * T1p);
+		    T1t = FNMS(KP634393284, T1s, KP773010453 * T1r);
+		    T1u = T1q + T1t;
+		    T2v = T1t - T1q;
+		    T1w = FMA(KP634393284, T1r, KP773010453 * T1s);
+		    T1x = FNMS(KP634393284, T1o, KP773010453 * T1p);
+		    T1y = T1w - T1x;
+		    T2z = T1x + T1w;
+	       }
+	       ro[WS(ros, 12)] = T1n - T1u;
+	       io[WS(ios, 12)] = T2z - T2A;
+	       ro[WS(ros, 3)] = T1n + T1u;
+	       io[WS(ios, 3)] = T2z + T2A;
+	       ro[WS(ros, 11)] = T1v - T1y;
+	       io[WS(ios, 11)] = T2v - T2y;
+	       ro[WS(ros, 4)] = T1v + T1y;
+	       io[WS(ios, 4)] = T2v + T2y;
+	  }
+	  {
+	       E T1L, T21, T2G, T2I, T20, T2H, T24, T2B;
+	       {
+		    E T1D, T1K, T2E, T2F;
+		    T1D = T1z - T1C;
+		    T1K = T1G - T1J;
+		    T1L = T1D + T1K;
+		    T21 = T1D - T1K;
+		    T2E = T2C - T2D;
+		    T2F = T26 + T27;
+		    T2G = T2E - T2F;
+		    T2I = T2F + T2E;
+	       }
+	       {
+		    E T1S, T1Z, T22, T23;
+		    T1S = FMA(KP881921264, T1O, KP471396736 * T1R);
+		    T1Z = FMA(KP881921264, T1V, KP471396736 * T1Y);
+		    T20 = T1S - T1Z;
+		    T2H = T1S + T1Z;
+		    T22 = FNMS(KP471396736, T1V, KP881921264 * T1Y);
+		    T23 = FNMS(KP471396736, T1O, KP881921264 * T1R);
+		    T24 = T22 - T23;
+		    T2B = T23 + T22;
+	       }
+	       ro[WS(ros, 13)] = T1L - T20;
+	       io[WS(ios, 13)] = T2B - T2G;
+	       ro[WS(ros, 2)] = T1L + T20;
+	       io[WS(ios, 2)] = T2B + T2G;
+	       ro[WS(ros, 10)] = T21 - T24;
+	       io[WS(ios, 10)] = T2I - T2H;
+	       ro[WS(ros, 5)] = T21 + T24;
+	       io[WS(ios, 5)] = -(T2H + T2I);
+	  }
+     }
+}
+
+static void mr2hcII_32(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  mr2hcII_32_0(I, ro, io, is, ros, ios);
+	  I += ivs;
+	  ro += ovs;
+	  io += ovs;
+     }
+}
+
+static const kr2hc_desc desc = { 32, "mr2hcII_32", {138, 46, 36, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_mr2hcII_32) (planner *p) {
+     X(kr2hcII_register) (p, mr2hcII_32, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/mr2hcII_64.c b/src/fftw3/rdft/codelets/r2hc/mr2hcII_64.c
new file mode 100644
index 0000000..4752322
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/mr2hcII_64.c
@@ -0,0 +1,799 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:59:51 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc_noinline -compact -variables 4 -n 64 -name mr2hcII_64 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 434 FP additions, 206 FP multiplications,
+ * (or, 342 additions, 114 multiplications, 92 fused multiply/add),
+ * 117 stack variables, and 128 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: mr2hcII_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: mr2hcII_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: mr2hcII_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void mr2hcII_64_0(const R *I, R *ro, R *io, stride is, stride ros, stride ios)
+{
+     DK(KP242980179, +0.242980179903263889948274162077471118320990783);
+     DK(KP970031253, +0.970031253194543992603984207286100251456865962);
+     DK(KP857728610, +0.857728610000272069902269984284770137042490799);
+     DK(KP514102744, +0.514102744193221726593693838968815772608049120);
+     DK(KP471396736, +0.471396736825997648556387625905254377657460319);
+     DK(KP881921264, +0.881921264348355029712756863660388349508442621);
+     DK(KP427555093, +0.427555093430282094320966856888798534304578629);
+     DK(KP903989293, +0.903989293123443331586200297230537048710132025);
+     DK(KP336889853, +0.336889853392220050689253212619147570477766780);
+     DK(KP941544065, +0.941544065183020778412509402599502357185589796);
+     DK(KP773010453, +0.773010453362736960810906609758469800971041293);
+     DK(KP634393284, +0.634393284163645498215171613225493370675687095);
+     DK(KP595699304, +0.595699304492433343467036528829969889511926338);
+     DK(KP803207531, +0.803207531480644909806676512963141923879569427);
+     DK(KP146730474, +0.146730474455361751658850129646717819706215317);
+     DK(KP989176509, +0.989176509964780973451673738016243063983689533);
+     DK(KP956940335, +0.956940335732208864935797886980269969482849206);
+     DK(KP290284677, +0.290284677254462367636192375817395274691476278);
+     DK(KP049067674, +0.049067674327418014254954976942682658314745363);
+     DK(KP998795456, +0.998795456205172392714771604759100694443203615);
+     DK(KP671558954, +0.671558954847018400625376850427421803228750632);
+     DK(KP740951125, +0.740951125354959091175616897495162729728955309);
+     DK(KP098017140, +0.098017140329560601994195563888641845861136673);
+     DK(KP995184726, +0.995184726672196886244836953109479921575474869);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     {
+	  E Tm, T34, T3Z, T5g, Tv, T35, T3W, T5h, Td, T33, T6B, T6Q, T3T, T5f, T68;
+	  E T6m, T2b, T3n, T4O, T5D, T2F, T3r, T4K, T5z, TK, T3c, T47, T5n, TR, T3b;
+	  E T44, T5o, T15, T38, T4e, T5l, T1c, T39, T4b, T5k, T1s, T3g, T4v, T5w, T1W;
+	  E T3k, T4k, T5s, T2u, T3q, T4R, T5A, T2y, T3o, T4H, T5C, T1L, T3j, T4y, T5t;
+	  E T1P, T3h, T4r, T5v;
+	  {
+	       E Te, Tk, Th, Tj, Tf, Tg;
+	       Te = I[WS(is, 4)];
+	       Tk = I[WS(is, 36)];
+	       Tf = I[WS(is, 20)];
+	       Tg = I[WS(is, 52)];
+	       Th = KP707106781 * (Tf - Tg);
+	       Tj = KP707106781 * (Tf + Tg);
+	       {
+		    E Ti, Tl, T3X, T3Y;
+		    Ti = Te + Th;
+		    Tl = Tj + Tk;
+		    Tm = FNMS(KP195090322, Tl, KP980785280 * Ti);
+		    T34 = FMA(KP195090322, Ti, KP980785280 * Tl);
+		    T3X = Tk - Tj;
+		    T3Y = Te - Th;
+		    T3Z = FNMS(KP555570233, T3Y, KP831469612 * T3X);
+		    T5g = FMA(KP831469612, T3Y, KP555570233 * T3X);
+	       }
+	  }
+	  {
+	       E Tq, Tt, Tp, Ts, Tn, To;
+	       Tq = I[WS(is, 60)];
+	       Tt = I[WS(is, 28)];
+	       Tn = I[WS(is, 12)];
+	       To = I[WS(is, 44)];
+	       Tp = KP707106781 * (Tn - To);
+	       Ts = KP707106781 * (Tn + To);
+	       {
+		    E Tr, Tu, T3U, T3V;
+		    Tr = Tp - Tq;
+		    Tu = Ts + Tt;
+		    Tv = FMA(KP980785280, Tr, KP195090322 * Tu);
+		    T35 = FNMS(KP980785280, Tu, KP195090322 * Tr);
+		    T3U = Tt - Ts;
+		    T3V = Tp + Tq;
+		    T3W = FNMS(KP555570233, T3V, KP831469612 * T3U);
+		    T5h = FMA(KP831469612, T3V, KP555570233 * T3U);
+	       }
+	  }
+	  {
+	       E T1, T66, T4, T65, T8, T3Q, Tb, T3R, T2, T3;
+	       T1 = I[0];
+	       T66 = I[WS(is, 32)];
+	       T2 = I[WS(is, 16)];
+	       T3 = I[WS(is, 48)];
+	       T4 = KP707106781 * (T2 - T3);
+	       T65 = KP707106781 * (T2 + T3);
+	       {
+		    E T6, T7, T9, Ta;
+		    T6 = I[WS(is, 8)];
+		    T7 = I[WS(is, 40)];
+		    T8 = FNMS(KP382683432, T7, KP923879532 * T6);
+		    T3Q = FMA(KP382683432, T6, KP923879532 * T7);
+		    T9 = I[WS(is, 24)];
+		    Ta = I[WS(is, 56)];
+		    Tb = FNMS(KP923879532, Ta, KP382683432 * T9);
+		    T3R = FMA(KP923879532, T9, KP382683432 * Ta);
+	       }
+	       {
+		    E T5, Tc, T6z, T6A;
+		    T5 = T1 + T4;
+		    Tc = T8 + Tb;
+		    Td = T5 + Tc;
+		    T33 = T5 - Tc;
+		    T6z = Tb - T8;
+		    T6A = T66 - T65;
+		    T6B = T6z - T6A;
+		    T6Q = T6z + T6A;
+	       }
+	       {
+		    E T3P, T3S, T64, T67;
+		    T3P = T1 - T4;
+		    T3S = T3Q - T3R;
+		    T3T = T3P - T3S;
+		    T5f = T3P + T3S;
+		    T64 = T3Q + T3R;
+		    T67 = T65 + T66;
+		    T68 = T64 + T67;
+		    T6m = T67 - T64;
+	       }
+	  }
+	  {
+	       E T22, T2D, T21, T2C, T26, T2z, T29, T2A, T1Z, T20;
+	       T22 = I[WS(is, 63)];
+	       T2D = I[WS(is, 31)];
+	       T1Z = I[WS(is, 15)];
+	       T20 = I[WS(is, 47)];
+	       T21 = KP707106781 * (T1Z - T20);
+	       T2C = KP707106781 * (T1Z + T20);
+	       {
+		    E T24, T25, T27, T28;
+		    T24 = I[WS(is, 7)];
+		    T25 = I[WS(is, 39)];
+		    T26 = FNMS(KP382683432, T25, KP923879532 * T24);
+		    T2z = FMA(KP382683432, T24, KP923879532 * T25);
+		    T27 = I[WS(is, 23)];
+		    T28 = I[WS(is, 55)];
+		    T29 = FNMS(KP923879532, T28, KP382683432 * T27);
+		    T2A = FMA(KP923879532, T27, KP382683432 * T28);
+	       }
+	       {
+		    E T23, T2a, T4M, T4N;
+		    T23 = T21 - T22;
+		    T2a = T26 + T29;
+		    T2b = T23 + T2a;
+		    T3n = T23 - T2a;
+		    T4M = T29 - T26;
+		    T4N = T2D - T2C;
+		    T4O = T4M - T4N;
+		    T5D = T4M + T4N;
+	       }
+	       {
+		    E T2B, T2E, T4I, T4J;
+		    T2B = T2z + T2A;
+		    T2E = T2C + T2D;
+		    T2F = T2B + T2E;
+		    T3r = T2E - T2B;
+		    T4I = T21 + T22;
+		    T4J = T2z - T2A;
+		    T4K = T4I + T4J;
+		    T5z = T4J - T4I;
+	       }
+	  }
+	  {
+	       E Ty, TP, TB, TO, TF, TL, TI, TM, Tz, TA;
+	       Ty = I[WS(is, 2)];
+	       TP = I[WS(is, 34)];
+	       Tz = I[WS(is, 18)];
+	       TA = I[WS(is, 50)];
+	       TB = KP707106781 * (Tz - TA);
+	       TO = KP707106781 * (Tz + TA);
+	       {
+		    E TD, TE, TG, TH;
+		    TD = I[WS(is, 10)];
+		    TE = I[WS(is, 42)];
+		    TF = FNMS(KP382683432, TE, KP923879532 * TD);
+		    TL = FMA(KP382683432, TD, KP923879532 * TE);
+		    TG = I[WS(is, 26)];
+		    TH = I[WS(is, 58)];
+		    TI = FNMS(KP923879532, TH, KP382683432 * TG);
+		    TM = FMA(KP923879532, TG, KP382683432 * TH);
+	       }
+	       {
+		    E TC, TJ, T45, T46;
+		    TC = Ty + TB;
+		    TJ = TF + TI;
+		    TK = TC + TJ;
+		    T3c = TC - TJ;
+		    T45 = TI - TF;
+		    T46 = TP - TO;
+		    T47 = T45 - T46;
+		    T5n = T45 + T46;
+	       }
+	       {
+		    E TN, TQ, T42, T43;
+		    TN = TL + TM;
+		    TQ = TO + TP;
+		    TR = TN + TQ;
+		    T3b = TQ - TN;
+		    T42 = Ty - TB;
+		    T43 = TL - TM;
+		    T44 = T42 - T43;
+		    T5o = T42 + T43;
+	       }
+	  }
+	  {
+	       E TW, T1a, TV, T19, T10, T16, T13, T17, TT, TU;
+	       TW = I[WS(is, 62)];
+	       T1a = I[WS(is, 30)];
+	       TT = I[WS(is, 14)];
+	       TU = I[WS(is, 46)];
+	       TV = KP707106781 * (TT - TU);
+	       T19 = KP707106781 * (TT + TU);
+	       {
+		    E TY, TZ, T11, T12;
+		    TY = I[WS(is, 6)];
+		    TZ = I[WS(is, 38)];
+		    T10 = FNMS(KP382683432, TZ, KP923879532 * TY);
+		    T16 = FMA(KP382683432, TY, KP923879532 * TZ);
+		    T11 = I[WS(is, 22)];
+		    T12 = I[WS(is, 54)];
+		    T13 = FNMS(KP923879532, T12, KP382683432 * T11);
+		    T17 = FMA(KP923879532, T11, KP382683432 * T12);
+	       }
+	       {
+		    E TX, T14, T4c, T4d;
+		    TX = TV - TW;
+		    T14 = T10 + T13;
+		    T15 = TX + T14;
+		    T38 = TX - T14;
+		    T4c = T13 - T10;
+		    T4d = T1a - T19;
+		    T4e = T4c - T4d;
+		    T5l = T4c + T4d;
+	       }
+	       {
+		    E T18, T1b, T49, T4a;
+		    T18 = T16 + T17;
+		    T1b = T19 + T1a;
+		    T1c = T18 + T1b;
+		    T39 = T1b - T18;
+		    T49 = TV + TW;
+		    T4a = T16 - T17;
+		    T4b = T49 + T4a;
+		    T5k = T4a - T49;
+	       }
+	  }
+	  {
+	       E T1g, T1U, T1j, T1T, T1n, T1Q, T1q, T1R, T1h, T1i;
+	       T1g = I[WS(is, 1)];
+	       T1U = I[WS(is, 33)];
+	       T1h = I[WS(is, 17)];
+	       T1i = I[WS(is, 49)];
+	       T1j = KP707106781 * (T1h - T1i);
+	       T1T = KP707106781 * (T1h + T1i);
+	       {
+		    E T1l, T1m, T1o, T1p;
+		    T1l = I[WS(is, 9)];
+		    T1m = I[WS(is, 41)];
+		    T1n = FNMS(KP382683432, T1m, KP923879532 * T1l);
+		    T1Q = FMA(KP382683432, T1l, KP923879532 * T1m);
+		    T1o = I[WS(is, 25)];
+		    T1p = I[WS(is, 57)];
+		    T1q = FNMS(KP923879532, T1p, KP382683432 * T1o);
+		    T1R = FMA(KP923879532, T1o, KP382683432 * T1p);
+	       }
+	       {
+		    E T1k, T1r, T4t, T4u;
+		    T1k = T1g + T1j;
+		    T1r = T1n + T1q;
+		    T1s = T1k + T1r;
+		    T3g = T1k - T1r;
+		    T4t = T1q - T1n;
+		    T4u = T1U - T1T;
+		    T4v = T4t - T4u;
+		    T5w = T4t + T4u;
+	       }
+	       {
+		    E T1S, T1V, T4i, T4j;
+		    T1S = T1Q + T1R;
+		    T1V = T1T + T1U;
+		    T1W = T1S + T1V;
+		    T3k = T1V - T1S;
+		    T4i = T1g - T1j;
+		    T4j = T1Q - T1R;
+		    T4k = T4i - T4j;
+		    T5s = T4i + T4j;
+	       }
+	  }
+	  {
+	       E T2g, T4F, T2j, T4E, T2p, T4C, T2s, T4B;
+	       {
+		    E T2c, T2i, T2f, T2h, T2d, T2e;
+		    T2c = I[WS(is, 3)];
+		    T2i = I[WS(is, 35)];
+		    T2d = I[WS(is, 19)];
+		    T2e = I[WS(is, 51)];
+		    T2f = KP707106781 * (T2d - T2e);
+		    T2h = KP707106781 * (T2d + T2e);
+		    T2g = T2c + T2f;
+		    T4F = T2c - T2f;
+		    T2j = T2h + T2i;
+		    T4E = T2i - T2h;
+	       }
+	       {
+		    E T2o, T2r, T2n, T2q, T2l, T2m;
+		    T2o = I[WS(is, 59)];
+		    T2r = I[WS(is, 27)];
+		    T2l = I[WS(is, 11)];
+		    T2m = I[WS(is, 43)];
+		    T2n = KP707106781 * (T2l - T2m);
+		    T2q = KP707106781 * (T2l + T2m);
+		    T2p = T2n - T2o;
+		    T4C = T2n + T2o;
+		    T2s = T2q + T2r;
+		    T4B = T2r - T2q;
+	       }
+	       {
+		    E T2k, T2t, T4P, T4Q;
+		    T2k = FNMS(KP195090322, T2j, KP980785280 * T2g);
+		    T2t = FMA(KP980785280, T2p, KP195090322 * T2s);
+		    T2u = T2k + T2t;
+		    T3q = T2t - T2k;
+		    T4P = FMA(KP831469612, T4F, KP555570233 * T4E);
+		    T4Q = FMA(KP831469612, T4C, KP555570233 * T4B);
+		    T4R = T4P + T4Q;
+		    T5A = T4P - T4Q;
+	       }
+	       {
+		    E T2w, T2x, T4D, T4G;
+		    T2w = FNMS(KP980785280, T2s, KP195090322 * T2p);
+		    T2x = FMA(KP195090322, T2g, KP980785280 * T2j);
+		    T2y = T2w - T2x;
+		    T3o = T2x + T2w;
+		    T4D = FNMS(KP555570233, T4C, KP831469612 * T4B);
+		    T4G = FNMS(KP555570233, T4F, KP831469612 * T4E);
+		    T4H = T4D - T4G;
+		    T5C = T4G + T4D;
+	       }
+	  }
+	  {
+	       E T1x, T4p, T1A, T4o, T1G, T4m, T1J, T4l;
+	       {
+		    E T1t, T1z, T1w, T1y, T1u, T1v;
+		    T1t = I[WS(is, 5)];
+		    T1z = I[WS(is, 37)];
+		    T1u = I[WS(is, 21)];
+		    T1v = I[WS(is, 53)];
+		    T1w = KP707106781 * (T1u - T1v);
+		    T1y = KP707106781 * (T1u + T1v);
+		    T1x = T1t + T1w;
+		    T4p = T1t - T1w;
+		    T1A = T1y + T1z;
+		    T4o = T1z - T1y;
+	       }
+	       {
+		    E T1F, T1I, T1E, T1H, T1C, T1D;
+		    T1F = I[WS(is, 61)];
+		    T1I = I[WS(is, 29)];
+		    T1C = I[WS(is, 13)];
+		    T1D = I[WS(is, 45)];
+		    T1E = KP707106781 * (T1C - T1D);
+		    T1H = KP707106781 * (T1C + T1D);
+		    T1G = T1E - T1F;
+		    T4m = T1E + T1F;
+		    T1J = T1H + T1I;
+		    T4l = T1I - T1H;
+	       }
+	       {
+		    E T1B, T1K, T4w, T4x;
+		    T1B = FNMS(KP195090322, T1A, KP980785280 * T1x);
+		    T1K = FMA(KP980785280, T1G, KP195090322 * T1J);
+		    T1L = T1B + T1K;
+		    T3j = T1K - T1B;
+		    T4w = FMA(KP831469612, T4p, KP555570233 * T4o);
+		    T4x = FMA(KP831469612, T4m, KP555570233 * T4l);
+		    T4y = T4w + T4x;
+		    T5t = T4w - T4x;
+	       }
+	       {
+		    E T1N, T1O, T4n, T4q;
+		    T1N = FNMS(KP980785280, T1J, KP195090322 * T1G);
+		    T1O = FMA(KP195090322, T1x, KP980785280 * T1A);
+		    T1P = T1N - T1O;
+		    T3h = T1O + T1N;
+		    T4n = FNMS(KP555570233, T4m, KP831469612 * T4l);
+		    T4q = FNMS(KP555570233, T4p, KP831469612 * T4o);
+		    T4r = T4n - T4q;
+		    T5v = T4q + T4n;
+	       }
+	  }
+	  {
+	       E Tx, T2N, T69, T6f, T1e, T6e, T2X, T30, T1Y, T2L, T2Q, T62, T2U, T31, T2H;
+	       E T2K, Tw, T63;
+	       Tw = Tm + Tv;
+	       Tx = Td + Tw;
+	       T2N = Td - Tw;
+	       T63 = T35 - T34;
+	       T69 = T63 - T68;
+	       T6f = T63 + T68;
+	       {
+		    E TS, T1d, T2V, T2W;
+		    TS = FNMS(KP098017140, TR, KP995184726 * TK);
+		    T1d = FMA(KP995184726, T15, KP098017140 * T1c);
+		    T1e = TS + T1d;
+		    T6e = T1d - TS;
+		    T2V = T2b - T2u;
+		    T2W = T2y + T2F;
+		    T2X = FNMS(KP671558954, T2W, KP740951125 * T2V);
+		    T30 = FMA(KP671558954, T2V, KP740951125 * T2W);
+	       }
+	       {
+		    E T1M, T1X, T2O, T2P;
+		    T1M = T1s + T1L;
+		    T1X = T1P - T1W;
+		    T1Y = FMA(KP998795456, T1M, KP049067674 * T1X);
+		    T2L = FNMS(KP049067674, T1M, KP998795456 * T1X);
+		    T2O = FMA(KP098017140, TK, KP995184726 * TR);
+		    T2P = FNMS(KP995184726, T1c, KP098017140 * T15);
+		    T2Q = T2O + T2P;
+		    T62 = T2P - T2O;
+	       }
+	       {
+		    E T2S, T2T, T2v, T2G;
+		    T2S = T1s - T1L;
+		    T2T = T1P + T1W;
+		    T2U = FMA(KP740951125, T2S, KP671558954 * T2T);
+		    T31 = FNMS(KP671558954, T2S, KP740951125 * T2T);
+		    T2v = T2b + T2u;
+		    T2G = T2y - T2F;
+		    T2H = FNMS(KP049067674, T2G, KP998795456 * T2v);
+		    T2K = FMA(KP049067674, T2v, KP998795456 * T2G);
+	       }
+	       {
+		    E T1f, T2I, T6b, T6c;
+		    T1f = Tx + T1e;
+		    T2I = T1Y + T2H;
+		    ro[WS(ros, 31)] = T1f - T2I;
+		    ro[0] = T1f + T2I;
+		    T6b = T2L + T2K;
+		    T6c = T62 + T69;
+		    io[WS(ios, 31)] = T6b - T6c;
+		    io[0] = T6b + T6c;
+	       }
+	       {
+		    E T2J, T2M, T61, T6a;
+		    T2J = Tx - T1e;
+		    T2M = T2K - T2L;
+		    ro[WS(ros, 16)] = T2J - T2M;
+		    ro[WS(ros, 15)] = T2J + T2M;
+		    T61 = T2H - T1Y;
+		    T6a = T62 - T69;
+		    io[WS(ios, 16)] = T61 - T6a;
+		    io[WS(ios, 15)] = T61 + T6a;
+	       }
+	       {
+		    E T2R, T2Y, T6h, T6i;
+		    T2R = T2N + T2Q;
+		    T2Y = T2U + T2X;
+		    ro[WS(ros, 24)] = T2R - T2Y;
+		    ro[WS(ros, 7)] = T2R + T2Y;
+		    T6h = T31 + T30;
+		    T6i = T6e + T6f;
+		    io[WS(ios, 24)] = T6h - T6i;
+		    io[WS(ios, 7)] = T6h + T6i;
+	       }
+	       {
+		    E T2Z, T32, T6d, T6g;
+		    T2Z = T2N - T2Q;
+		    T32 = T30 - T31;
+		    ro[WS(ros, 23)] = T2Z - T32;
+		    ro[WS(ros, 8)] = T2Z + T32;
+		    T6d = T2X - T2U;
+		    T6g = T6e - T6f;
+		    io[WS(ios, 23)] = T6d - T6g;
+		    io[WS(ios, 8)] = T6d + T6g;
+	       }
+	  }
+	  {
+	       E T5j, T5L, T6R, T6X, T5q, T6W, T5V, T5Y, T5y, T5J, T5O, T6O, T5S, T5Z, T5F;
+	       E T5I, T5i, T6P;
+	       T5i = T5g - T5h;
+	       T5j = T5f - T5i;
+	       T5L = T5f + T5i;
+	       T6P = T3Z + T3W;
+	       T6R = T6P - T6Q;
+	       T6X = T6P + T6Q;
+	       {
+		    E T5m, T5p, T5T, T5U;
+		    T5m = FMA(KP290284677, T5k, KP956940335 * T5l);
+		    T5p = FNMS(KP290284677, T5o, KP956940335 * T5n);
+		    T5q = T5m - T5p;
+		    T6W = T5p + T5m;
+		    T5T = T5z + T5A;
+		    T5U = T5C + T5D;
+		    T5V = FNMS(KP146730474, T5U, KP989176509 * T5T);
+		    T5Y = FMA(KP146730474, T5T, KP989176509 * T5U);
+	       }
+	       {
+		    E T5u, T5x, T5M, T5N;
+		    T5u = T5s - T5t;
+		    T5x = T5v - T5w;
+		    T5y = FMA(KP803207531, T5u, KP595699304 * T5x);
+		    T5J = FNMS(KP595699304, T5u, KP803207531 * T5x);
+		    T5M = FMA(KP956940335, T5o, KP290284677 * T5n);
+		    T5N = FNMS(KP290284677, T5l, KP956940335 * T5k);
+		    T5O = T5M + T5N;
+		    T6O = T5N - T5M;
+	       }
+	       {
+		    E T5Q, T5R, T5B, T5E;
+		    T5Q = T5s + T5t;
+		    T5R = T5v + T5w;
+		    T5S = FMA(KP989176509, T5Q, KP146730474 * T5R);
+		    T5Z = FNMS(KP146730474, T5Q, KP989176509 * T5R);
+		    T5B = T5z - T5A;
+		    T5E = T5C - T5D;
+		    T5F = FNMS(KP595699304, T5E, KP803207531 * T5B);
+		    T5I = FMA(KP595699304, T5B, KP803207531 * T5E);
+	       }
+	       {
+		    E T5r, T5G, T6T, T6U;
+		    T5r = T5j + T5q;
+		    T5G = T5y + T5F;
+		    ro[WS(ros, 25)] = T5r - T5G;
+		    ro[WS(ros, 6)] = T5r + T5G;
+		    T6T = T5J + T5I;
+		    T6U = T6O + T6R;
+		    io[WS(ios, 25)] = T6T - T6U;
+		    io[WS(ios, 6)] = T6T + T6U;
+	       }
+	       {
+		    E T5H, T5K, T6N, T6S;
+		    T5H = T5j - T5q;
+		    T5K = T5I - T5J;
+		    ro[WS(ros, 22)] = T5H - T5K;
+		    ro[WS(ros, 9)] = T5H + T5K;
+		    T6N = T5F - T5y;
+		    T6S = T6O - T6R;
+		    io[WS(ios, 22)] = T6N - T6S;
+		    io[WS(ios, 9)] = T6N + T6S;
+	       }
+	       {
+		    E T5P, T5W, T6Z, T70;
+		    T5P = T5L + T5O;
+		    T5W = T5S + T5V;
+		    ro[WS(ros, 30)] = T5P - T5W;
+		    ro[WS(ros, 1)] = T5P + T5W;
+		    T6Z = T5Z + T5Y;
+		    T70 = T6W + T6X;
+		    io[WS(ios, 30)] = T6Z - T70;
+		    io[WS(ios, 1)] = T6Z + T70;
+	       }
+	       {
+		    E T5X, T60, T6V, T6Y;
+		    T5X = T5L - T5O;
+		    T60 = T5Y - T5Z;
+		    ro[WS(ros, 17)] = T5X - T60;
+		    ro[WS(ros, 14)] = T5X + T60;
+		    T6V = T5V - T5S;
+		    T6Y = T6W - T6X;
+		    io[WS(ios, 17)] = T6V - T6Y;
+		    io[WS(ios, 14)] = T6V + T6Y;
+	       }
+	  }
+	  {
+	       E T37, T3z, T6n, T6t, T3e, T6s, T3J, T3M, T3m, T3x, T3C, T6k, T3G, T3N, T3t;
+	       E T3w, T36, T6l;
+	       T36 = T34 + T35;
+	       T37 = T33 - T36;
+	       T3z = T33 + T36;
+	       T6l = Tv - Tm;
+	       T6n = T6l - T6m;
+	       T6t = T6l + T6m;
+	       {
+		    E T3a, T3d, T3H, T3I;
+		    T3a = FMA(KP634393284, T38, KP773010453 * T39);
+		    T3d = FNMS(KP634393284, T3c, KP773010453 * T3b);
+		    T3e = T3a - T3d;
+		    T6s = T3d + T3a;
+		    T3H = T3n + T3o;
+		    T3I = T3q + T3r;
+		    T3J = FNMS(KP336889853, T3I, KP941544065 * T3H);
+		    T3M = FMA(KP336889853, T3H, KP941544065 * T3I);
+	       }
+	       {
+		    E T3i, T3l, T3A, T3B;
+		    T3i = T3g - T3h;
+		    T3l = T3j - T3k;
+		    T3m = FMA(KP903989293, T3i, KP427555093 * T3l);
+		    T3x = FNMS(KP427555093, T3i, KP903989293 * T3l);
+		    T3A = FMA(KP773010453, T3c, KP634393284 * T3b);
+		    T3B = FNMS(KP634393284, T39, KP773010453 * T38);
+		    T3C = T3A + T3B;
+		    T6k = T3B - T3A;
+	       }
+	       {
+		    E T3E, T3F, T3p, T3s;
+		    T3E = T3g + T3h;
+		    T3F = T3j + T3k;
+		    T3G = FMA(KP941544065, T3E, KP336889853 * T3F);
+		    T3N = FNMS(KP336889853, T3E, KP941544065 * T3F);
+		    T3p = T3n - T3o;
+		    T3s = T3q - T3r;
+		    T3t = FNMS(KP427555093, T3s, KP903989293 * T3p);
+		    T3w = FMA(KP427555093, T3p, KP903989293 * T3s);
+	       }
+	       {
+		    E T3f, T3u, T6p, T6q;
+		    T3f = T37 + T3e;
+		    T3u = T3m + T3t;
+		    ro[WS(ros, 27)] = T3f - T3u;
+		    ro[WS(ros, 4)] = T3f + T3u;
+		    T6p = T3x + T3w;
+		    T6q = T6k + T6n;
+		    io[WS(ios, 27)] = T6p - T6q;
+		    io[WS(ios, 4)] = T6p + T6q;
+	       }
+	       {
+		    E T3v, T3y, T6j, T6o;
+		    T3v = T37 - T3e;
+		    T3y = T3w - T3x;
+		    ro[WS(ros, 20)] = T3v - T3y;
+		    ro[WS(ros, 11)] = T3v + T3y;
+		    T6j = T3t - T3m;
+		    T6o = T6k - T6n;
+		    io[WS(ios, 20)] = T6j - T6o;
+		    io[WS(ios, 11)] = T6j + T6o;
+	       }
+	       {
+		    E T3D, T3K, T6v, T6w;
+		    T3D = T3z + T3C;
+		    T3K = T3G + T3J;
+		    ro[WS(ros, 28)] = T3D - T3K;
+		    ro[WS(ros, 3)] = T3D + T3K;
+		    T6v = T3N + T3M;
+		    T6w = T6s + T6t;
+		    io[WS(ios, 28)] = T6v - T6w;
+		    io[WS(ios, 3)] = T6v + T6w;
+	       }
+	       {
+		    E T3L, T3O, T6r, T6u;
+		    T3L = T3z - T3C;
+		    T3O = T3M - T3N;
+		    ro[WS(ros, 19)] = T3L - T3O;
+		    ro[WS(ros, 12)] = T3L + T3O;
+		    T6r = T3J - T3G;
+		    T6u = T6s - T6t;
+		    io[WS(ios, 19)] = T6r - T6u;
+		    io[WS(ios, 12)] = T6r + T6u;
+	       }
+	  }
+	  {
+	       E T41, T4Z, T6D, T6J, T4g, T6I, T59, T5d, T4A, T4X, T52, T6y, T56, T5c, T4T;
+	       E T4W, T40, T6C;
+	       T40 = T3W - T3Z;
+	       T41 = T3T + T40;
+	       T4Z = T3T - T40;
+	       T6C = T5g + T5h;
+	       T6D = T6B - T6C;
+	       T6J = T6C + T6B;
+	       {
+		    E T48, T4f, T57, T58;
+		    T48 = FMA(KP881921264, T44, KP471396736 * T47);
+		    T4f = FMA(KP881921264, T4b, KP471396736 * T4e);
+		    T4g = T48 - T4f;
+		    T6I = T48 + T4f;
+		    T57 = T4K + T4H;
+		    T58 = T4R + T4O;
+		    T59 = FMA(KP514102744, T57, KP857728610 * T58);
+		    T5d = FNMS(KP857728610, T57, KP514102744 * T58);
+	       }
+	       {
+		    E T4s, T4z, T50, T51;
+		    T4s = T4k + T4r;
+		    T4z = T4v - T4y;
+		    T4A = FMA(KP970031253, T4s, KP242980179 * T4z);
+		    T4X = FNMS(KP242980179, T4s, KP970031253 * T4z);
+		    T50 = FNMS(KP471396736, T4b, KP881921264 * T4e);
+		    T51 = FNMS(KP471396736, T44, KP881921264 * T47);
+		    T52 = T50 - T51;
+		    T6y = T51 + T50;
+	       }
+	       {
+		    E T54, T55, T4L, T4S;
+		    T54 = T4k - T4r;
+		    T55 = T4y + T4v;
+		    T56 = FMA(KP514102744, T54, KP857728610 * T55);
+		    T5c = FNMS(KP514102744, T55, KP857728610 * T54);
+		    T4L = T4H - T4K;
+		    T4S = T4O - T4R;
+		    T4T = FNMS(KP242980179, T4S, KP970031253 * T4L);
+		    T4W = FMA(KP242980179, T4L, KP970031253 * T4S);
+	       }
+	       {
+		    E T4h, T4U, T6F, T6G;
+		    T4h = T41 + T4g;
+		    T4U = T4A + T4T;
+		    ro[WS(ros, 29)] = T4h - T4U;
+		    ro[WS(ros, 2)] = T4h + T4U;
+		    T6F = T4X + T4W;
+		    T6G = T6y + T6D;
+		    io[WS(ios, 29)] = T6F - T6G;
+		    io[WS(ios, 2)] = T6F + T6G;
+	       }
+	       {
+		    E T4V, T4Y, T6x, T6E;
+		    T4V = T41 - T4g;
+		    T4Y = T4W - T4X;
+		    ro[WS(ros, 18)] = T4V - T4Y;
+		    ro[WS(ros, 13)] = T4V + T4Y;
+		    T6x = T4T - T4A;
+		    T6E = T6y - T6D;
+		    io[WS(ios, 18)] = T6x - T6E;
+		    io[WS(ios, 13)] = T6x + T6E;
+	       }
+	       {
+		    E T53, T5a, T6L, T6M;
+		    T53 = T4Z - T52;
+		    T5a = T56 - T59;
+		    ro[WS(ros, 21)] = T53 - T5a;
+		    ro[WS(ros, 10)] = T53 + T5a;
+		    T6L = T5d - T5c;
+		    T6M = T6J - T6I;
+		    io[WS(ios, 21)] = T6L - T6M;
+		    io[WS(ios, 10)] = T6L + T6M;
+	       }
+	       {
+		    E T5b, T5e, T6H, T6K;
+		    T5b = T4Z + T52;
+		    T5e = T5c + T5d;
+		    ro[WS(ros, 26)] = T5b - T5e;
+		    ro[WS(ros, 5)] = T5b + T5e;
+		    T6H = T56 + T59;
+		    T6K = T6I + T6J;
+		    io[WS(ios, 5)] = -(T6H + T6K);
+		    io[WS(ios, 26)] = T6K - T6H;
+	       }
+	  }
+     }
+}
+
+static void mr2hcII_64(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  mr2hcII_64_0(I, ro, io, is, ros, ios);
+	  I += ivs;
+	  ro += ovs;
+	  io += ovs;
+     }
+}
+
+static const kr2hc_desc desc = { 64, "mr2hcII_64", {342, 114, 92, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_mr2hcII_64) (planner *p) {
+     X(kr2hcII_register) (p, mr2hcII_64, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/mr2hc_128.c b/src/fftw3/rdft/codelets/r2hc/mr2hc_128.c
new file mode 100644
index 0000000..0409559
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/mr2hc_128.c
@@ -0,0 +1,1647 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:49 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc_noinline -compact -variables 4 -n 128 -name mr2hc_128 -include r2hc.h */
+
+/*
+ * This function contains 956 FP additions, 330 FP multiplications,
+ * (or, 812 additions, 186 multiplications, 144 fused multiply/add),
+ * 185 stack variables, and 256 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: mr2hc_128.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: mr2hc_128.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: mr2hc_128.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void mr2hc_128_0(const R *I, R *ro, R *io, stride is, stride ros, stride ios)
+{
+     DK(KP803207531, +0.803207531480644909806676512963141923879569427);
+     DK(KP595699304, +0.595699304492433343467036528829969889511926338);
+     DK(KP146730474, +0.146730474455361751658850129646717819706215317);
+     DK(KP989176509, +0.989176509964780973451673738016243063983689533);
+     DK(KP740951125, +0.740951125354959091175616897495162729728955309);
+     DK(KP671558954, +0.671558954847018400625376850427421803228750632);
+     DK(KP049067674, +0.049067674327418014254954976942682658314745363);
+     DK(KP998795456, +0.998795456205172392714771604759100694443203615);
+     DK(KP242980179, +0.242980179903263889948274162077471118320990783);
+     DK(KP970031253, +0.970031253194543992603984207286100251456865962);
+     DK(KP514102744, +0.514102744193221726593693838968815772608049120);
+     DK(KP857728610, +0.857728610000272069902269984284770137042490799);
+     DK(KP336889853, +0.336889853392220050689253212619147570477766780);
+     DK(KP941544065, +0.941544065183020778412509402599502357185589796);
+     DK(KP427555093, +0.427555093430282094320966856888798534304578629);
+     DK(KP903989293, +0.903989293123443331586200297230537048710132025);
+     DK(KP098017140, +0.098017140329560601994195563888641845861136673);
+     DK(KP995184726, +0.995184726672196886244836953109479921575474869);
+     DK(KP634393284, +0.634393284163645498215171613225493370675687095);
+     DK(KP773010453, +0.773010453362736960810906609758469800971041293);
+     DK(KP881921264, +0.881921264348355029712756863660388349508442621);
+     DK(KP471396736, +0.471396736825997648556387625905254377657460319);
+     DK(KP956940335, +0.956940335732208864935797886980269969482849206);
+     DK(KP290284677, +0.290284677254462367636192375817395274691476278);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     {
+	  E TcD, TdU, T27, T7r, T5S, T8y, Tf, Ta5, Tu, Tbq, TcG, TdV, T2e, T8z, T5V;
+	  E T7s, TK, Ta6, TcK, TdX, T2o, T5X, T7w, T8B, TZ, Ta7, TcN, TdY, T2x, T5Y;
+	  E T7z, T8C, T1g, Taa, TcU, TeA, TcX, Tez, T1v, Tab, T2M, T6z, T7E, T9e, T7H;
+	  E T9d, T2T, T6A, T4X, T6L, Tdz, TeL, TdK, TeP, T5G, T6P, T8d, T9p, TaV, Tc3;
+	  E Tbi, Tc4, T8o, T9t, T3I, T6H, Tde, TeH, Tdp, TeF, T4r, T6F, T7U, T9l, Tao;
+	  E TbW, TaL, TbX, T85, T9j, T1L, Tad, Td3, Tew, Td6, Tex, T20, Tae, T37, T6x;
+	  E T7L, T9a, T7O, T9b, T3e, T6w, TbZ, Tc0, T3Z, T4s, Tds, TeI, T4g, T4t, T80;
+	  E T87, Tdl, TeE, T7X, T86, TaD, TaM, Tc6, Tc7, T5e, T5H, TdN, TeM, T5v, T5I;
+	  E T8j, T8q, TdG, TeO, T8g, T8p, Tba, Tbj;
+	  {
+	       E T3, T23, Td, T25, T6, T5R, Ta, T24;
+	       {
+		    E T1, T2, Tb, Tc;
+		    T1 = I[0];
+		    T2 = I[WS(is, 64)];
+		    T3 = T1 + T2;
+		    T23 = T1 - T2;
+		    Tb = I[WS(is, 112)];
+		    Tc = I[WS(is, 48)];
+		    Td = Tb + Tc;
+		    T25 = Tb - Tc;
+	       }
+	       {
+		    E T4, T5, T8, T9;
+		    T4 = I[WS(is, 32)];
+		    T5 = I[WS(is, 96)];
+		    T6 = T4 + T5;
+		    T5R = T4 - T5;
+		    T8 = I[WS(is, 16)];
+		    T9 = I[WS(is, 80)];
+		    Ta = T8 + T9;
+		    T24 = T8 - T9;
+	       }
+	       TcD = T3 - T6;
+	       TdU = Td - Ta;
+	       {
+		    E T26, T5Q, T7, Te;
+		    T26 = KP707106781 * (T24 + T25);
+		    T27 = T23 + T26;
+		    T7r = T23 - T26;
+		    T5Q = KP707106781 * (T25 - T24);
+		    T5S = T5Q - T5R;
+		    T8y = T5R + T5Q;
+		    T7 = T3 + T6;
+		    Te = Ta + Td;
+		    Tf = T7 + Te;
+		    Ta5 = T7 - Te;
+	       }
+	  }
+	  {
+	       E Ti, T28, Ts, T2c, Tl, T29, Tp, T2b;
+	       {
+		    E Tg, Th, Tq, Tr;
+		    Tg = I[WS(is, 8)];
+		    Th = I[WS(is, 72)];
+		    Ti = Tg + Th;
+		    T28 = Tg - Th;
+		    Tq = I[WS(is, 24)];
+		    Tr = I[WS(is, 88)];
+		    Ts = Tq + Tr;
+		    T2c = Tq - Tr;
+	       }
+	       {
+		    E Tj, Tk, Tn, To;
+		    Tj = I[WS(is, 40)];
+		    Tk = I[WS(is, 104)];
+		    Tl = Tj + Tk;
+		    T29 = Tj - Tk;
+		    Tn = I[WS(is, 120)];
+		    To = I[WS(is, 56)];
+		    Tp = Tn + To;
+		    T2b = Tn - To;
+	       }
+	       {
+		    E Tm, Tt, TcE, TcF;
+		    Tm = Ti + Tl;
+		    Tt = Tp + Ts;
+		    Tu = Tm + Tt;
+		    Tbq = Tt - Tm;
+		    TcE = Ti - Tl;
+		    TcF = Tp - Ts;
+		    TcG = KP707106781 * (TcE + TcF);
+		    TdV = KP707106781 * (TcF - TcE);
+	       }
+	       {
+		    E T2a, T2d, T5T, T5U;
+		    T2a = FNMS(KP382683432, T29, KP923879532 * T28);
+		    T2d = FMA(KP923879532, T2b, KP382683432 * T2c);
+		    T2e = T2a + T2d;
+		    T8z = T2d - T2a;
+		    T5T = FNMS(KP923879532, T2c, KP382683432 * T2b);
+		    T5U = FMA(KP382683432, T28, KP923879532 * T29);
+		    T5V = T5T - T5U;
+		    T7s = T5U + T5T;
+	       }
+	  }
+	  {
+	       E Ty, T2g, TB, T2m, TF, T2l, TI, T2j;
+	       {
+		    E Tw, Tx, Tz, TA;
+		    Tw = I[WS(is, 4)];
+		    Tx = I[WS(is, 68)];
+		    Ty = Tw + Tx;
+		    T2g = Tw - Tx;
+		    Tz = I[WS(is, 36)];
+		    TA = I[WS(is, 100)];
+		    TB = Tz + TA;
+		    T2m = Tz - TA;
+		    {
+			 E TD, TE, T2h, TG, TH, T2i;
+			 TD = I[WS(is, 20)];
+			 TE = I[WS(is, 84)];
+			 T2h = TD - TE;
+			 TG = I[WS(is, 116)];
+			 TH = I[WS(is, 52)];
+			 T2i = TG - TH;
+			 TF = TD + TE;
+			 T2l = KP707106781 * (T2i - T2h);
+			 TI = TG + TH;
+			 T2j = KP707106781 * (T2h + T2i);
+		    }
+	       }
+	       {
+		    E TC, TJ, TcI, TcJ;
+		    TC = Ty + TB;
+		    TJ = TF + TI;
+		    TK = TC + TJ;
+		    Ta6 = TC - TJ;
+		    TcI = Ty - TB;
+		    TcJ = TI - TF;
+		    TcK = FMA(KP923879532, TcI, KP382683432 * TcJ);
+		    TdX = FNMS(KP382683432, TcI, KP923879532 * TcJ);
+	       }
+	       {
+		    E T2k, T2n, T7u, T7v;
+		    T2k = T2g + T2j;
+		    T2n = T2l - T2m;
+		    T2o = FMA(KP980785280, T2k, KP195090322 * T2n);
+		    T5X = FNMS(KP195090322, T2k, KP980785280 * T2n);
+		    T7u = T2g - T2j;
+		    T7v = T2m + T2l;
+		    T7w = FMA(KP831469612, T7u, KP555570233 * T7v);
+		    T8B = FNMS(KP555570233, T7u, KP831469612 * T7v);
+	       }
+	  }
+	  {
+	       E TN, T2p, TQ, T2v, TU, T2u, TX, T2s;
+	       {
+		    E TL, TM, TO, TP;
+		    TL = I[WS(is, 124)];
+		    TM = I[WS(is, 60)];
+		    TN = TL + TM;
+		    T2p = TL - TM;
+		    TO = I[WS(is, 28)];
+		    TP = I[WS(is, 92)];
+		    TQ = TO + TP;
+		    T2v = TO - TP;
+		    {
+			 E TS, TT, T2q, TV, TW, T2r;
+			 TS = I[WS(is, 12)];
+			 TT = I[WS(is, 76)];
+			 T2q = TS - TT;
+			 TV = I[WS(is, 108)];
+			 TW = I[WS(is, 44)];
+			 T2r = TV - TW;
+			 TU = TS + TT;
+			 T2u = KP707106781 * (T2r - T2q);
+			 TX = TV + TW;
+			 T2s = KP707106781 * (T2q + T2r);
+		    }
+	       }
+	       {
+		    E TR, TY, TcL, TcM;
+		    TR = TN + TQ;
+		    TY = TU + TX;
+		    TZ = TR + TY;
+		    Ta7 = TR - TY;
+		    TcL = TN - TQ;
+		    TcM = TX - TU;
+		    TcN = FNMS(KP382683432, TcM, KP923879532 * TcL);
+		    TdY = FMA(KP382683432, TcL, KP923879532 * TcM);
+	       }
+	       {
+		    E T2t, T2w, T7x, T7y;
+		    T2t = T2p + T2s;
+		    T2w = T2u - T2v;
+		    T2x = FNMS(KP195090322, T2w, KP980785280 * T2t);
+		    T5Y = FMA(KP195090322, T2t, KP980785280 * T2w);
+		    T7x = T2p - T2s;
+		    T7y = T2v + T2u;
+		    T7z = FNMS(KP555570233, T7y, KP831469612 * T7x);
+		    T8C = FMA(KP555570233, T7x, KP831469612 * T7y);
+	       }
+	  }
+	  {
+	       E T14, T2N, T17, T2D, T1b, T2O, T1e, T2C, T1j, T1m, T2K, TcR, T2Q, T1q, T1t;
+	       E T2H, TcS, T2R;
+	       {
+		    E T12, T13, T15, T16;
+		    T12 = I[WS(is, 2)];
+		    T13 = I[WS(is, 66)];
+		    T14 = T12 + T13;
+		    T2N = T12 - T13;
+		    T15 = I[WS(is, 34)];
+		    T16 = I[WS(is, 98)];
+		    T17 = T15 + T16;
+		    T2D = T15 - T16;
+	       }
+	       {
+		    E T19, T1a, T2B, T1c, T1d, T2A;
+		    T19 = I[WS(is, 18)];
+		    T1a = I[WS(is, 82)];
+		    T2B = T19 - T1a;
+		    T1c = I[WS(is, 114)];
+		    T1d = I[WS(is, 50)];
+		    T2A = T1c - T1d;
+		    T1b = T19 + T1a;
+		    T2O = KP707106781 * (T2B + T2A);
+		    T1e = T1c + T1d;
+		    T2C = KP707106781 * (T2A - T2B);
+	       }
+	       {
+		    E T2I, T2J, T2F, T2G;
+		    {
+			 E T1h, T1i, T1k, T1l;
+			 T1h = I[WS(is, 10)];
+			 T1i = I[WS(is, 74)];
+			 T1j = T1h + T1i;
+			 T2I = T1h - T1i;
+			 T1k = I[WS(is, 42)];
+			 T1l = I[WS(is, 106)];
+			 T1m = T1k + T1l;
+			 T2J = T1k - T1l;
+		    }
+		    T2K = FMA(KP382683432, T2I, KP923879532 * T2J);
+		    TcR = T1j - T1m;
+		    T2Q = FNMS(KP382683432, T2J, KP923879532 * T2I);
+		    {
+			 E T1o, T1p, T1r, T1s;
+			 T1o = I[WS(is, 122)];
+			 T1p = I[WS(is, 58)];
+			 T1q = T1o + T1p;
+			 T2F = T1o - T1p;
+			 T1r = I[WS(is, 26)];
+			 T1s = I[WS(is, 90)];
+			 T1t = T1r + T1s;
+			 T2G = T1r - T1s;
+		    }
+		    T2H = FNMS(KP923879532, T2G, KP382683432 * T2F);
+		    TcS = T1q - T1t;
+		    T2R = FMA(KP923879532, T2F, KP382683432 * T2G);
+	       }
+	       {
+		    E T18, T1f, TcQ, TcT;
+		    T18 = T14 + T17;
+		    T1f = T1b + T1e;
+		    T1g = T18 + T1f;
+		    Taa = T18 - T1f;
+		    TcQ = T14 - T17;
+		    TcT = KP707106781 * (TcR + TcS);
+		    TcU = TcQ + TcT;
+		    TeA = TcQ - TcT;
+	       }
+	       {
+		    E TcV, TcW, T1n, T1u;
+		    TcV = T1e - T1b;
+		    TcW = KP707106781 * (TcS - TcR);
+		    TcX = TcV + TcW;
+		    Tez = TcW - TcV;
+		    T1n = T1j + T1m;
+		    T1u = T1q + T1t;
+		    T1v = T1n + T1u;
+		    Tab = T1u - T1n;
+	       }
+	       {
+		    E T2E, T2L, T7C, T7D;
+		    T2E = T2C - T2D;
+		    T2L = T2H - T2K;
+		    T2M = T2E + T2L;
+		    T6z = T2L - T2E;
+		    T7C = T2N - T2O;
+		    T7D = T2K + T2H;
+		    T7E = T7C + T7D;
+		    T9e = T7C - T7D;
+	       }
+	       {
+		    E T7F, T7G, T2P, T2S;
+		    T7F = T2D + T2C;
+		    T7G = T2R - T2Q;
+		    T7H = T7F + T7G;
+		    T9d = T7G - T7F;
+		    T2P = T2N + T2O;
+		    T2S = T2Q + T2R;
+		    T2T = T2P + T2S;
+		    T6A = T2P - T2S;
+	       }
+	  }
+	  {
+	       E T4z, TaP, T5B, TaQ, T4G, TaT, T5y, TaS, Tbf, Tbg, T4O, Tdw, T5E, Tbc, Tbd;
+	       E T4V, Tdx, T5D;
+	       {
+		    E T4x, T4y, T5z, T5A;
+		    T4x = I[WS(is, 127)];
+		    T4y = I[WS(is, 63)];
+		    T4z = T4x - T4y;
+		    TaP = T4x + T4y;
+		    T5z = I[WS(is, 31)];
+		    T5A = I[WS(is, 95)];
+		    T5B = T5z - T5A;
+		    TaQ = T5z + T5A;
+	       }
+	       {
+		    E T4A, T4B, T4C, T4D, T4E, T4F;
+		    T4A = I[WS(is, 15)];
+		    T4B = I[WS(is, 79)];
+		    T4C = T4A - T4B;
+		    T4D = I[WS(is, 111)];
+		    T4E = I[WS(is, 47)];
+		    T4F = T4D - T4E;
+		    T4G = KP707106781 * (T4C + T4F);
+		    TaT = T4D + T4E;
+		    T5y = KP707106781 * (T4F - T4C);
+		    TaS = T4A + T4B;
+	       }
+	       {
+		    E T4K, T4N, T4R, T4U;
+		    {
+			 E T4I, T4J, T4L, T4M;
+			 T4I = I[WS(is, 7)];
+			 T4J = I[WS(is, 71)];
+			 T4K = T4I - T4J;
+			 Tbf = T4I + T4J;
+			 T4L = I[WS(is, 39)];
+			 T4M = I[WS(is, 103)];
+			 T4N = T4L - T4M;
+			 Tbg = T4L + T4M;
+		    }
+		    T4O = FNMS(KP382683432, T4N, KP923879532 * T4K);
+		    Tdw = Tbf - Tbg;
+		    T5E = FMA(KP382683432, T4K, KP923879532 * T4N);
+		    {
+			 E T4P, T4Q, T4S, T4T;
+			 T4P = I[WS(is, 119)];
+			 T4Q = I[WS(is, 55)];
+			 T4R = T4P - T4Q;
+			 Tbc = T4P + T4Q;
+			 T4S = I[WS(is, 23)];
+			 T4T = I[WS(is, 87)];
+			 T4U = T4S - T4T;
+			 Tbd = T4S + T4T;
+		    }
+		    T4V = FMA(KP923879532, T4R, KP382683432 * T4U);
+		    Tdx = Tbc - Tbd;
+		    T5D = FNMS(KP923879532, T4U, KP382683432 * T4R);
+	       }
+	       {
+		    E T4H, T4W, Tdv, Tdy;
+		    T4H = T4z + T4G;
+		    T4W = T4O + T4V;
+		    T4X = T4H + T4W;
+		    T6L = T4H - T4W;
+		    Tdv = TaP - TaQ;
+		    Tdy = KP707106781 * (Tdw + Tdx);
+		    Tdz = Tdv + Tdy;
+		    TeL = Tdv - Tdy;
+	       }
+	       {
+		    E TdI, TdJ, T5C, T5F;
+		    TdI = TaT - TaS;
+		    TdJ = KP707106781 * (Tdx - Tdw);
+		    TdK = TdI + TdJ;
+		    TeP = TdJ - TdI;
+		    T5C = T5y - T5B;
+		    T5F = T5D - T5E;
+		    T5G = T5C + T5F;
+		    T6P = T5F - T5C;
+	       }
+	       {
+		    E T8b, T8c, TaR, TaU;
+		    T8b = T4z - T4G;
+		    T8c = T5E + T5D;
+		    T8d = T8b + T8c;
+		    T9p = T8b - T8c;
+		    TaR = TaP + TaQ;
+		    TaU = TaS + TaT;
+		    TaV = TaR - TaU;
+		    Tc3 = TaR + TaU;
+	       }
+	       {
+		    E Tbe, Tbh, T8m, T8n;
+		    Tbe = Tbc + Tbd;
+		    Tbh = Tbf + Tbg;
+		    Tbi = Tbe - Tbh;
+		    Tc4 = Tbh + Tbe;
+		    T8m = T5B + T5y;
+		    T8n = T4V - T4O;
+		    T8o = T8m + T8n;
+		    T9t = T8n - T8m;
+	       }
+	  }
+	  {
+	       E T3k, Tai, T4m, Taj, T3r, Tam, T4j, Tal, TaI, TaJ, T3z, Tdb, T4p, TaF, TaG;
+	       E T3G, Tdc, T4o;
+	       {
+		    E T3i, T3j, T4k, T4l;
+		    T3i = I[WS(is, 1)];
+		    T3j = I[WS(is, 65)];
+		    T3k = T3i - T3j;
+		    Tai = T3i + T3j;
+		    T4k = I[WS(is, 33)];
+		    T4l = I[WS(is, 97)];
+		    T4m = T4k - T4l;
+		    Taj = T4k + T4l;
+	       }
+	       {
+		    E T3l, T3m, T3n, T3o, T3p, T3q;
+		    T3l = I[WS(is, 17)];
+		    T3m = I[WS(is, 81)];
+		    T3n = T3l - T3m;
+		    T3o = I[WS(is, 113)];
+		    T3p = I[WS(is, 49)];
+		    T3q = T3o - T3p;
+		    T3r = KP707106781 * (T3n + T3q);
+		    Tam = T3o + T3p;
+		    T4j = KP707106781 * (T3q - T3n);
+		    Tal = T3l + T3m;
+	       }
+	       {
+		    E T3v, T3y, T3C, T3F;
+		    {
+			 E T3t, T3u, T3w, T3x;
+			 T3t = I[WS(is, 9)];
+			 T3u = I[WS(is, 73)];
+			 T3v = T3t - T3u;
+			 TaI = T3t + T3u;
+			 T3w = I[WS(is, 41)];
+			 T3x = I[WS(is, 105)];
+			 T3y = T3w - T3x;
+			 TaJ = T3w + T3x;
+		    }
+		    T3z = FNMS(KP382683432, T3y, KP923879532 * T3v);
+		    Tdb = TaI - TaJ;
+		    T4p = FMA(KP382683432, T3v, KP923879532 * T3y);
+		    {
+			 E T3A, T3B, T3D, T3E;
+			 T3A = I[WS(is, 121)];
+			 T3B = I[WS(is, 57)];
+			 T3C = T3A - T3B;
+			 TaF = T3A + T3B;
+			 T3D = I[WS(is, 25)];
+			 T3E = I[WS(is, 89)];
+			 T3F = T3D - T3E;
+			 TaG = T3D + T3E;
+		    }
+		    T3G = FMA(KP923879532, T3C, KP382683432 * T3F);
+		    Tdc = TaF - TaG;
+		    T4o = FNMS(KP923879532, T3F, KP382683432 * T3C);
+	       }
+	       {
+		    E T3s, T3H, Tda, Tdd;
+		    T3s = T3k + T3r;
+		    T3H = T3z + T3G;
+		    T3I = T3s + T3H;
+		    T6H = T3s - T3H;
+		    Tda = Tai - Taj;
+		    Tdd = KP707106781 * (Tdb + Tdc);
+		    Tde = Tda + Tdd;
+		    TeH = Tda - Tdd;
+	       }
+	       {
+		    E Tdn, Tdo, T4n, T4q;
+		    Tdn = Tam - Tal;
+		    Tdo = KP707106781 * (Tdc - Tdb);
+		    Tdp = Tdn + Tdo;
+		    TeF = Tdo - Tdn;
+		    T4n = T4j - T4m;
+		    T4q = T4o - T4p;
+		    T4r = T4n + T4q;
+		    T6F = T4q - T4n;
+	       }
+	       {
+		    E T7S, T7T, Tak, Tan;
+		    T7S = T3k - T3r;
+		    T7T = T4p + T4o;
+		    T7U = T7S + T7T;
+		    T9l = T7S - T7T;
+		    Tak = Tai + Taj;
+		    Tan = Tal + Tam;
+		    Tao = Tak - Tan;
+		    TbW = Tak + Tan;
+	       }
+	       {
+		    E TaH, TaK, T83, T84;
+		    TaH = TaF + TaG;
+		    TaK = TaI + TaJ;
+		    TaL = TaH - TaK;
+		    TbX = TaK + TaH;
+		    T83 = T4m + T4j;
+		    T84 = T3G - T3z;
+		    T85 = T83 + T84;
+		    T9j = T84 - T83;
+	       }
+	  }
+	  {
+	       E T1z, T2V, T1C, T39, T1G, T38, T1J, T2Y, T1O, T1R, T32, Td0, T3c, T1V, T1Y;
+	       E T35, Td1, T3b;
+	       {
+		    E T1x, T1y, T1A, T1B;
+		    T1x = I[WS(is, 126)];
+		    T1y = I[WS(is, 62)];
+		    T1z = T1x + T1y;
+		    T2V = T1x - T1y;
+		    T1A = I[WS(is, 30)];
+		    T1B = I[WS(is, 94)];
+		    T1C = T1A + T1B;
+		    T39 = T1A - T1B;
+	       }
+	       {
+		    E T1E, T1F, T2W, T1H, T1I, T2X;
+		    T1E = I[WS(is, 14)];
+		    T1F = I[WS(is, 78)];
+		    T2W = T1E - T1F;
+		    T1H = I[WS(is, 110)];
+		    T1I = I[WS(is, 46)];
+		    T2X = T1H - T1I;
+		    T1G = T1E + T1F;
+		    T38 = KP707106781 * (T2X - T2W);
+		    T1J = T1H + T1I;
+		    T2Y = KP707106781 * (T2W + T2X);
+	       }
+	       {
+		    E T30, T31, T33, T34;
+		    {
+			 E T1M, T1N, T1P, T1Q;
+			 T1M = I[WS(is, 6)];
+			 T1N = I[WS(is, 70)];
+			 T1O = T1M + T1N;
+			 T30 = T1M - T1N;
+			 T1P = I[WS(is, 38)];
+			 T1Q = I[WS(is, 102)];
+			 T1R = T1P + T1Q;
+			 T31 = T1P - T1Q;
+		    }
+		    T32 = FNMS(KP382683432, T31, KP923879532 * T30);
+		    Td0 = T1O - T1R;
+		    T3c = FMA(KP382683432, T30, KP923879532 * T31);
+		    {
+			 E T1T, T1U, T1W, T1X;
+			 T1T = I[WS(is, 118)];
+			 T1U = I[WS(is, 54)];
+			 T1V = T1T + T1U;
+			 T33 = T1T - T1U;
+			 T1W = I[WS(is, 22)];
+			 T1X = I[WS(is, 86)];
+			 T1Y = T1W + T1X;
+			 T34 = T1W - T1X;
+		    }
+		    T35 = FMA(KP923879532, T33, KP382683432 * T34);
+		    Td1 = T1V - T1Y;
+		    T3b = FNMS(KP923879532, T34, KP382683432 * T33);
+	       }
+	       {
+		    E T1D, T1K, TcZ, Td2;
+		    T1D = T1z + T1C;
+		    T1K = T1G + T1J;
+		    T1L = T1D + T1K;
+		    Tad = T1D - T1K;
+		    TcZ = T1z - T1C;
+		    Td2 = KP707106781 * (Td0 + Td1);
+		    Td3 = TcZ + Td2;
+		    Tew = TcZ - Td2;
+	       }
+	       {
+		    E Td4, Td5, T1S, T1Z;
+		    Td4 = T1J - T1G;
+		    Td5 = KP707106781 * (Td1 - Td0);
+		    Td6 = Td4 + Td5;
+		    Tex = Td5 - Td4;
+		    T1S = T1O + T1R;
+		    T1Z = T1V + T1Y;
+		    T20 = T1S + T1Z;
+		    Tae = T1Z - T1S;
+	       }
+	       {
+		    E T2Z, T36, T7J, T7K;
+		    T2Z = T2V + T2Y;
+		    T36 = T32 + T35;
+		    T37 = T2Z + T36;
+		    T6x = T2Z - T36;
+		    T7J = T2V - T2Y;
+		    T7K = T3c + T3b;
+		    T7L = T7J + T7K;
+		    T9a = T7J - T7K;
+	       }
+	       {
+		    E T7M, T7N, T3a, T3d;
+		    T7M = T39 + T38;
+		    T7N = T35 - T32;
+		    T7O = T7M + T7N;
+		    T9b = T7N - T7M;
+		    T3a = T38 - T39;
+		    T3d = T3b - T3c;
+		    T3e = T3a + T3d;
+		    T6w = T3d - T3a;
+	       }
+	  }
+	  {
+	       E T3L, Tdf, T3X, Tar, T42, Tdi, T4e, Tay, T3S, Tdg, T3U, Tau, T49, Tdj, T4b;
+	       E TaB, Tdh, Tdk;
+	       {
+		    E T3J, T3K, Tap, T3V, T3W, Taq;
+		    T3J = I[WS(is, 5)];
+		    T3K = I[WS(is, 69)];
+		    Tap = T3J + T3K;
+		    T3V = I[WS(is, 37)];
+		    T3W = I[WS(is, 101)];
+		    Taq = T3V + T3W;
+		    T3L = T3J - T3K;
+		    Tdf = Tap - Taq;
+		    T3X = T3V - T3W;
+		    Tar = Tap + Taq;
+	       }
+	       {
+		    E T40, T41, Taw, T4c, T4d, Tax;
+		    T40 = I[WS(is, 125)];
+		    T41 = I[WS(is, 61)];
+		    Taw = T40 + T41;
+		    T4c = I[WS(is, 29)];
+		    T4d = I[WS(is, 93)];
+		    Tax = T4c + T4d;
+		    T42 = T40 - T41;
+		    Tdi = Taw - Tax;
+		    T4e = T4c - T4d;
+		    Tay = Taw + Tax;
+	       }
+	       {
+		    E T3O, Tas, T3R, Tat;
+		    {
+			 E T3M, T3N, T3P, T3Q;
+			 T3M = I[WS(is, 21)];
+			 T3N = I[WS(is, 85)];
+			 T3O = T3M - T3N;
+			 Tas = T3M + T3N;
+			 T3P = I[WS(is, 117)];
+			 T3Q = I[WS(is, 53)];
+			 T3R = T3P - T3Q;
+			 Tat = T3P + T3Q;
+		    }
+		    T3S = KP707106781 * (T3O + T3R);
+		    Tdg = Tat - Tas;
+		    T3U = KP707106781 * (T3R - T3O);
+		    Tau = Tas + Tat;
+	       }
+	       {
+		    E T45, Taz, T48, TaA;
+		    {
+			 E T43, T44, T46, T47;
+			 T43 = I[WS(is, 13)];
+			 T44 = I[WS(is, 77)];
+			 T45 = T43 - T44;
+			 Taz = T43 + T44;
+			 T46 = I[WS(is, 109)];
+			 T47 = I[WS(is, 45)];
+			 T48 = T46 - T47;
+			 TaA = T46 + T47;
+		    }
+		    T49 = KP707106781 * (T45 + T48);
+		    Tdj = TaA - Taz;
+		    T4b = KP707106781 * (T48 - T45);
+		    TaB = Taz + TaA;
+	       }
+	       TbZ = Tar + Tau;
+	       Tc0 = Tay + TaB;
+	       {
+		    E T3T, T3Y, Tdq, Tdr;
+		    T3T = T3L + T3S;
+		    T3Y = T3U - T3X;
+		    T3Z = FMA(KP980785280, T3T, KP195090322 * T3Y);
+		    T4s = FNMS(KP195090322, T3T, KP980785280 * T3Y);
+		    Tdq = FNMS(KP382683432, Tdf, KP923879532 * Tdg);
+		    Tdr = FMA(KP382683432, Tdi, KP923879532 * Tdj);
+		    Tds = Tdq + Tdr;
+		    TeI = Tdr - Tdq;
+	       }
+	       {
+		    E T4a, T4f, T7Y, T7Z;
+		    T4a = T42 + T49;
+		    T4f = T4b - T4e;
+		    T4g = FNMS(KP195090322, T4f, KP980785280 * T4a);
+		    T4t = FMA(KP195090322, T4a, KP980785280 * T4f);
+		    T7Y = T42 - T49;
+		    T7Z = T4e + T4b;
+		    T80 = FNMS(KP555570233, T7Z, KP831469612 * T7Y);
+		    T87 = FMA(KP555570233, T7Y, KP831469612 * T7Z);
+	       }
+	       Tdh = FMA(KP923879532, Tdf, KP382683432 * Tdg);
+	       Tdk = FNMS(KP382683432, Tdj, KP923879532 * Tdi);
+	       Tdl = Tdh + Tdk;
+	       TeE = Tdk - Tdh;
+	       {
+		    E T7V, T7W, Tav, TaC;
+		    T7V = T3L - T3S;
+		    T7W = T3X + T3U;
+		    T7X = FMA(KP831469612, T7V, KP555570233 * T7W);
+		    T86 = FNMS(KP555570233, T7V, KP831469612 * T7W);
+		    Tav = Tar - Tau;
+		    TaC = Tay - TaB;
+		    TaD = KP707106781 * (Tav + TaC);
+		    TaM = KP707106781 * (TaC - Tav);
+	       }
+	  }
+	  {
+	       E T50, TdA, T5c, TaY, T5h, TdD, T5t, Tb5, T57, TdB, T59, Tb1, T5o, TdE, T5q;
+	       E Tb8, TdC, TdF;
+	       {
+		    E T4Y, T4Z, TaW, T5a, T5b, TaX;
+		    T4Y = I[WS(is, 3)];
+		    T4Z = I[WS(is, 67)];
+		    TaW = T4Y + T4Z;
+		    T5a = I[WS(is, 35)];
+		    T5b = I[WS(is, 99)];
+		    TaX = T5a + T5b;
+		    T50 = T4Y - T4Z;
+		    TdA = TaW - TaX;
+		    T5c = T5a - T5b;
+		    TaY = TaW + TaX;
+	       }
+	       {
+		    E T5f, T5g, Tb3, T5r, T5s, Tb4;
+		    T5f = I[WS(is, 123)];
+		    T5g = I[WS(is, 59)];
+		    Tb3 = T5f + T5g;
+		    T5r = I[WS(is, 27)];
+		    T5s = I[WS(is, 91)];
+		    Tb4 = T5r + T5s;
+		    T5h = T5f - T5g;
+		    TdD = Tb3 - Tb4;
+		    T5t = T5r - T5s;
+		    Tb5 = Tb3 + Tb4;
+	       }
+	       {
+		    E T53, TaZ, T56, Tb0;
+		    {
+			 E T51, T52, T54, T55;
+			 T51 = I[WS(is, 19)];
+			 T52 = I[WS(is, 83)];
+			 T53 = T51 - T52;
+			 TaZ = T51 + T52;
+			 T54 = I[WS(is, 115)];
+			 T55 = I[WS(is, 51)];
+			 T56 = T54 - T55;
+			 Tb0 = T54 + T55;
+		    }
+		    T57 = KP707106781 * (T53 + T56);
+		    TdB = Tb0 - TaZ;
+		    T59 = KP707106781 * (T56 - T53);
+		    Tb1 = TaZ + Tb0;
+	       }
+	       {
+		    E T5k, Tb6, T5n, Tb7;
+		    {
+			 E T5i, T5j, T5l, T5m;
+			 T5i = I[WS(is, 11)];
+			 T5j = I[WS(is, 75)];
+			 T5k = T5i - T5j;
+			 Tb6 = T5i + T5j;
+			 T5l = I[WS(is, 107)];
+			 T5m = I[WS(is, 43)];
+			 T5n = T5l - T5m;
+			 Tb7 = T5l + T5m;
+		    }
+		    T5o = KP707106781 * (T5k + T5n);
+		    TdE = Tb7 - Tb6;
+		    T5q = KP707106781 * (T5n - T5k);
+		    Tb8 = Tb6 + Tb7;
+	       }
+	       Tc6 = TaY + Tb1;
+	       Tc7 = Tb5 + Tb8;
+	       {
+		    E T58, T5d, TdL, TdM;
+		    T58 = T50 + T57;
+		    T5d = T59 - T5c;
+		    T5e = FMA(KP980785280, T58, KP195090322 * T5d);
+		    T5H = FNMS(KP195090322, T58, KP980785280 * T5d);
+		    TdL = FNMS(KP382683432, TdA, KP923879532 * TdB);
+		    TdM = FMA(KP382683432, TdD, KP923879532 * TdE);
+		    TdN = TdL + TdM;
+		    TeM = TdM - TdL;
+	       }
+	       {
+		    E T5p, T5u, T8h, T8i;
+		    T5p = T5h + T5o;
+		    T5u = T5q - T5t;
+		    T5v = FNMS(KP195090322, T5u, KP980785280 * T5p);
+		    T5I = FMA(KP195090322, T5p, KP980785280 * T5u);
+		    T8h = T5h - T5o;
+		    T8i = T5t + T5q;
+		    T8j = FNMS(KP555570233, T8i, KP831469612 * T8h);
+		    T8q = FMA(KP555570233, T8h, KP831469612 * T8i);
+	       }
+	       TdC = FMA(KP923879532, TdA, KP382683432 * TdB);
+	       TdF = FNMS(KP382683432, TdE, KP923879532 * TdD);
+	       TdG = TdC + TdF;
+	       TeO = TdF - TdC;
+	       {
+		    E T8e, T8f, Tb2, Tb9;
+		    T8e = T50 - T57;
+		    T8f = T5c + T59;
+		    T8g = FMA(KP831469612, T8e, KP555570233 * T8f);
+		    T8p = FNMS(KP555570233, T8e, KP831469612 * T8f);
+		    Tb2 = TaY - Tb1;
+		    Tb9 = Tb5 - Tb8;
+		    Tba = KP707106781 * (Tb2 + Tb9);
+		    Tbj = KP707106781 * (Tb9 - Tb2);
+	       }
+	  }
+	  {
+	       E T11, TbV, Tc9, Tcf, T22, Tcb, Tc2, Tce;
+	       {
+		    E Tv, T10, Tc5, Tc8;
+		    Tv = Tf + Tu;
+		    T10 = TK + TZ;
+		    T11 = Tv + T10;
+		    TbV = Tv - T10;
+		    Tc5 = Tc3 + Tc4;
+		    Tc8 = Tc6 + Tc7;
+		    Tc9 = Tc5 - Tc8;
+		    Tcf = Tc5 + Tc8;
+	       }
+	       {
+		    E T1w, T21, TbY, Tc1;
+		    T1w = T1g + T1v;
+		    T21 = T1L + T20;
+		    T22 = T1w + T21;
+		    Tcb = T21 - T1w;
+		    TbY = TbW + TbX;
+		    Tc1 = TbZ + Tc0;
+		    Tc2 = TbY - Tc1;
+		    Tce = TbY + Tc1;
+	       }
+	       ro[WS(ros, 32)] = T11 - T22;
+	       io[WS(ios, 32)] = Tcf - Tce;
+	       {
+		    E Tca, Tcc, Tcd, Tcg;
+		    Tca = KP707106781 * (Tc2 + Tc9);
+		    ro[WS(ros, 48)] = TbV - Tca;
+		    ro[WS(ros, 16)] = TbV + Tca;
+		    Tcc = KP707106781 * (Tc9 - Tc2);
+		    io[WS(ios, 16)] = Tcb + Tcc;
+		    io[WS(ios, 48)] = Tcc - Tcb;
+		    Tcd = T11 + T22;
+		    Tcg = Tce + Tcf;
+		    ro[WS(ros, 64)] = Tcd - Tcg;
+		    ro[0] = Tcd + Tcg;
+	       }
+	  }
+	  {
+	       E Tch, Tcu, Tck, Tct, Tco, Tcy, Tcr, Tcz, Tci, Tcj;
+	       Tch = Tf - Tu;
+	       Tcu = TZ - TK;
+	       Tci = T1g - T1v;
+	       Tcj = T1L - T20;
+	       Tck = KP707106781 * (Tci + Tcj);
+	       Tct = KP707106781 * (Tcj - Tci);
+	       {
+		    E Tcm, Tcn, Tcp, Tcq;
+		    Tcm = TbW - TbX;
+		    Tcn = Tc0 - TbZ;
+		    Tco = FMA(KP923879532, Tcm, KP382683432 * Tcn);
+		    Tcy = FNMS(KP382683432, Tcm, KP923879532 * Tcn);
+		    Tcp = Tc3 - Tc4;
+		    Tcq = Tc7 - Tc6;
+		    Tcr = FNMS(KP382683432, Tcq, KP923879532 * Tcp);
+		    Tcz = FMA(KP382683432, Tcp, KP923879532 * Tcq);
+	       }
+	       {
+		    E Tcl, Tcs, Tcx, TcA;
+		    Tcl = Tch + Tck;
+		    Tcs = Tco + Tcr;
+		    ro[WS(ros, 56)] = Tcl - Tcs;
+		    ro[WS(ros, 8)] = Tcl + Tcs;
+		    Tcx = Tcu + Tct;
+		    TcA = Tcy + Tcz;
+		    io[WS(ios, 8)] = Tcx + TcA;
+		    io[WS(ios, 56)] = TcA - Tcx;
+	       }
+	       {
+		    E Tcv, Tcw, TcB, TcC;
+		    Tcv = Tct - Tcu;
+		    Tcw = Tcr - Tco;
+		    io[WS(ios, 24)] = Tcv + Tcw;
+		    io[WS(ios, 40)] = Tcw - Tcv;
+		    TcB = Tch - Tck;
+		    TcC = Tcz - Tcy;
+		    ro[WS(ros, 40)] = TcB - TcC;
+		    ro[WS(ros, 24)] = TcB + TcC;
+	       }
+	  }
+	  {
+	       E Ta9, TbB, Tbs, TbM, Tag, TbL, TbJ, TbR, TaO, Tbw, Tbp, TbC, TbG, TbQ, Tbl;
+	       E Tbx, Ta8, Tbr;
+	       Ta8 = KP707106781 * (Ta6 + Ta7);
+	       Ta9 = Ta5 + Ta8;
+	       TbB = Ta5 - Ta8;
+	       Tbr = KP707106781 * (Ta7 - Ta6);
+	       Tbs = Tbq + Tbr;
+	       TbM = Tbr - Tbq;
+	       {
+		    E Tac, Taf, TbH, TbI;
+		    Tac = FMA(KP923879532, Taa, KP382683432 * Tab);
+		    Taf = FNMS(KP382683432, Tae, KP923879532 * Tad);
+		    Tag = Tac + Taf;
+		    TbL = Taf - Tac;
+		    TbH = TaV - Tba;
+		    TbI = Tbj - Tbi;
+		    TbJ = FNMS(KP555570233, TbI, KP831469612 * TbH);
+		    TbR = FMA(KP555570233, TbH, KP831469612 * TbI);
+	       }
+	       {
+		    E TaE, TaN, Tbn, Tbo;
+		    TaE = Tao + TaD;
+		    TaN = TaL + TaM;
+		    TaO = FMA(KP980785280, TaE, KP195090322 * TaN);
+		    Tbw = FNMS(KP195090322, TaE, KP980785280 * TaN);
+		    Tbn = FNMS(KP382683432, Taa, KP923879532 * Tab);
+		    Tbo = FMA(KP382683432, Tad, KP923879532 * Tae);
+		    Tbp = Tbn + Tbo;
+		    TbC = Tbo - Tbn;
+	       }
+	       {
+		    E TbE, TbF, Tbb, Tbk;
+		    TbE = Tao - TaD;
+		    TbF = TaM - TaL;
+		    TbG = FMA(KP831469612, TbE, KP555570233 * TbF);
+		    TbQ = FNMS(KP555570233, TbE, KP831469612 * TbF);
+		    Tbb = TaV + Tba;
+		    Tbk = Tbi + Tbj;
+		    Tbl = FNMS(KP195090322, Tbk, KP980785280 * Tbb);
+		    Tbx = FMA(KP195090322, Tbb, KP980785280 * Tbk);
+	       }
+	       {
+		    E Tah, Tbm, Tbv, Tby;
+		    Tah = Ta9 + Tag;
+		    Tbm = TaO + Tbl;
+		    ro[WS(ros, 60)] = Tah - Tbm;
+		    ro[WS(ros, 4)] = Tah + Tbm;
+		    Tbv = Tbs + Tbp;
+		    Tby = Tbw + Tbx;
+		    io[WS(ios, 4)] = Tbv + Tby;
+		    io[WS(ios, 60)] = Tby - Tbv;
+	       }
+	       {
+		    E Tbt, Tbu, Tbz, TbA;
+		    Tbt = Tbp - Tbs;
+		    Tbu = Tbl - TaO;
+		    io[WS(ios, 28)] = Tbt + Tbu;
+		    io[WS(ios, 36)] = Tbu - Tbt;
+		    Tbz = Ta9 - Tag;
+		    TbA = Tbx - Tbw;
+		    ro[WS(ros, 36)] = Tbz - TbA;
+		    ro[WS(ros, 28)] = Tbz + TbA;
+	       }
+	       {
+		    E TbD, TbK, TbP, TbS;
+		    TbD = TbB + TbC;
+		    TbK = TbG + TbJ;
+		    ro[WS(ros, 52)] = TbD - TbK;
+		    ro[WS(ros, 12)] = TbD + TbK;
+		    TbP = TbM + TbL;
+		    TbS = TbQ + TbR;
+		    io[WS(ios, 12)] = TbP + TbS;
+		    io[WS(ios, 52)] = TbS - TbP;
+	       }
+	       {
+		    E TbN, TbO, TbT, TbU;
+		    TbN = TbL - TbM;
+		    TbO = TbJ - TbG;
+		    io[WS(ios, 20)] = TbN + TbO;
+		    io[WS(ios, 44)] = TbO - TbN;
+		    TbT = TbB - TbC;
+		    TbU = TbR - TbQ;
+		    ro[WS(ros, 44)] = TbT - TbU;
+		    ro[WS(ros, 20)] = TbT + TbU;
+	       }
+	  }
+	  {
+	       E Tev, Tf7, Tfc, Tfm, Tff, Tfn, TeC, Tfh, TeK, Tf2, TeV, Tf8, TeY, Tfi, TeR;
+	       E Tf3;
+	       {
+		    E Tet, Teu, Tfa, Tfb;
+		    Tet = TcD - TcG;
+		    Teu = TdY - TdX;
+		    Tev = Tet - Teu;
+		    Tf7 = Tet + Teu;
+		    Tfa = TeF + TeE;
+		    Tfb = TeH + TeI;
+		    Tfc = FMA(KP290284677, Tfa, KP956940335 * Tfb);
+		    Tfm = FNMS(KP290284677, Tfb, KP956940335 * Tfa);
+	       }
+	       {
+		    E Tfd, Tfe, Tey, TeB;
+		    Tfd = TeL + TeM;
+		    Tfe = TeP + TeO;
+		    Tff = FNMS(KP290284677, Tfe, KP956940335 * Tfd);
+		    Tfn = FMA(KP956940335, Tfe, KP290284677 * Tfd);
+		    Tey = FMA(KP555570233, Tew, KP831469612 * Tex);
+		    TeB = FNMS(KP555570233, TeA, KP831469612 * Tez);
+		    TeC = Tey - TeB;
+		    Tfh = TeB + Tey;
+	       }
+	       {
+		    E TeG, TeJ, TeT, TeU;
+		    TeG = TeE - TeF;
+		    TeJ = TeH - TeI;
+		    TeK = FMA(KP471396736, TeG, KP881921264 * TeJ);
+		    Tf2 = FNMS(KP471396736, TeJ, KP881921264 * TeG);
+		    TeT = FNMS(KP555570233, Tex, KP831469612 * Tew);
+		    TeU = FMA(KP831469612, TeA, KP555570233 * Tez);
+		    TeV = TeT - TeU;
+		    Tf8 = TeU + TeT;
+	       }
+	       {
+		    E TeW, TeX, TeN, TeQ;
+		    TeW = TcN - TcK;
+		    TeX = TdV - TdU;
+		    TeY = TeW - TeX;
+		    Tfi = TeX + TeW;
+		    TeN = TeL - TeM;
+		    TeQ = TeO - TeP;
+		    TeR = FNMS(KP471396736, TeQ, KP881921264 * TeN);
+		    Tf3 = FMA(KP881921264, TeQ, KP471396736 * TeN);
+	       }
+	       {
+		    E TeD, TeS, Tf1, Tf4;
+		    TeD = Tev + TeC;
+		    TeS = TeK + TeR;
+		    ro[WS(ros, 54)] = TeD - TeS;
+		    ro[WS(ros, 10)] = TeD + TeS;
+		    Tf1 = TeY + TeV;
+		    Tf4 = Tf2 + Tf3;
+		    io[WS(ios, 10)] = Tf1 + Tf4;
+		    io[WS(ios, 54)] = Tf4 - Tf1;
+	       }
+	       {
+		    E TeZ, Tf0, Tf5, Tf6;
+		    TeZ = TeV - TeY;
+		    Tf0 = TeR - TeK;
+		    io[WS(ios, 22)] = TeZ + Tf0;
+		    io[WS(ios, 42)] = Tf0 - TeZ;
+		    Tf5 = Tev - TeC;
+		    Tf6 = Tf3 - Tf2;
+		    ro[WS(ros, 42)] = Tf5 - Tf6;
+		    ro[WS(ros, 22)] = Tf5 + Tf6;
+	       }
+	       {
+		    E Tf9, Tfg, Tfl, Tfo;
+		    Tf9 = Tf7 + Tf8;
+		    Tfg = Tfc + Tff;
+		    ro[WS(ros, 58)] = Tf9 - Tfg;
+		    ro[WS(ros, 6)] = Tf9 + Tfg;
+		    Tfl = Tfi + Tfh;
+		    Tfo = Tfm + Tfn;
+		    io[WS(ios, 6)] = Tfl + Tfo;
+		    io[WS(ios, 58)] = Tfo - Tfl;
+	       }
+	       {
+		    E Tfj, Tfk, Tfp, Tfq;
+		    Tfj = Tfh - Tfi;
+		    Tfk = Tff - Tfc;
+		    io[WS(ios, 26)] = Tfj + Tfk;
+		    io[WS(ios, 38)] = Tfk - Tfj;
+		    Tfp = Tf7 - Tf8;
+		    Tfq = Tfn - Tfm;
+		    ro[WS(ros, 38)] = Tfp - Tfq;
+		    ro[WS(ros, 26)] = Tfp + Tfq;
+	       }
+	  }
+	  {
+	       E TcP, Te9, Tee, Teo, Teh, Tep, Td8, Tej, Tdu, Te4, TdT, Tea, Te0, Tek, TdP;
+	       E Te5;
+	       {
+		    E TcH, TcO, Tec, Ted;
+		    TcH = TcD + TcG;
+		    TcO = TcK + TcN;
+		    TcP = TcH + TcO;
+		    Te9 = TcH - TcO;
+		    Tec = Tde - Tdl;
+		    Ted = Tds - Tdp;
+		    Tee = FMA(KP773010453, Tec, KP634393284 * Ted);
+		    Teo = FNMS(KP634393284, Tec, KP773010453 * Ted);
+	       }
+	       {
+		    E Tef, Teg, TcY, Td7;
+		    Tef = Tdz - TdG;
+		    Teg = TdN - TdK;
+		    Teh = FNMS(KP634393284, Teg, KP773010453 * Tef);
+		    Tep = FMA(KP634393284, Tef, KP773010453 * Teg);
+		    TcY = FMA(KP980785280, TcU, KP195090322 * TcX);
+		    Td7 = FNMS(KP195090322, Td6, KP980785280 * Td3);
+		    Td8 = TcY + Td7;
+		    Tej = Td7 - TcY;
+	       }
+	       {
+		    E Tdm, Tdt, TdR, TdS;
+		    Tdm = Tde + Tdl;
+		    Tdt = Tdp + Tds;
+		    Tdu = FMA(KP995184726, Tdm, KP098017140 * Tdt);
+		    Te4 = FNMS(KP098017140, Tdm, KP995184726 * Tdt);
+		    TdR = FNMS(KP195090322, TcU, KP980785280 * TcX);
+		    TdS = FMA(KP195090322, Td3, KP980785280 * Td6);
+		    TdT = TdR + TdS;
+		    Tea = TdS - TdR;
+	       }
+	       {
+		    E TdW, TdZ, TdH, TdO;
+		    TdW = TdU + TdV;
+		    TdZ = TdX + TdY;
+		    Te0 = TdW + TdZ;
+		    Tek = TdZ - TdW;
+		    TdH = Tdz + TdG;
+		    TdO = TdK + TdN;
+		    TdP = FNMS(KP098017140, TdO, KP995184726 * TdH);
+		    Te5 = FMA(KP098017140, TdH, KP995184726 * TdO);
+	       }
+	       {
+		    E Td9, TdQ, Te3, Te6;
+		    Td9 = TcP + Td8;
+		    TdQ = Tdu + TdP;
+		    ro[WS(ros, 62)] = Td9 - TdQ;
+		    ro[WS(ros, 2)] = Td9 + TdQ;
+		    Te3 = Te0 + TdT;
+		    Te6 = Te4 + Te5;
+		    io[WS(ios, 2)] = Te3 + Te6;
+		    io[WS(ios, 62)] = Te6 - Te3;
+	       }
+	       {
+		    E Te1, Te2, Te7, Te8;
+		    Te1 = TdT - Te0;
+		    Te2 = TdP - Tdu;
+		    io[WS(ios, 30)] = Te1 + Te2;
+		    io[WS(ios, 34)] = Te2 - Te1;
+		    Te7 = TcP - Td8;
+		    Te8 = Te5 - Te4;
+		    ro[WS(ros, 34)] = Te7 - Te8;
+		    ro[WS(ros, 30)] = Te7 + Te8;
+	       }
+	       {
+		    E Teb, Tei, Ten, Teq;
+		    Teb = Te9 + Tea;
+		    Tei = Tee + Teh;
+		    ro[WS(ros, 50)] = Teb - Tei;
+		    ro[WS(ros, 14)] = Teb + Tei;
+		    Ten = Tek + Tej;
+		    Teq = Teo + Tep;
+		    io[WS(ios, 14)] = Ten + Teq;
+		    io[WS(ios, 50)] = Teq - Ten;
+	       }
+	       {
+		    E Tel, Tem, Ter, Tes;
+		    Tel = Tej - Tek;
+		    Tem = Teh - Tee;
+		    io[WS(ios, 18)] = Tel + Tem;
+		    io[WS(ios, 46)] = Tem - Tel;
+		    Ter = Te9 - Tea;
+		    Tes = Tep - Teo;
+		    ro[WS(ros, 46)] = Ter - Tes;
+		    ro[WS(ros, 18)] = Ter + Tes;
+	       }
+	  }
+	  {
+	       E T6v, T77, T6C, T7h, T6Y, T7i, T6V, T78, T6R, T7n, T73, T7f, T6K, T7m, T72;
+	       E T7c;
+	       {
+		    E T6t, T6u, T6T, T6U;
+		    T6t = T27 - T2e;
+		    T6u = T5Y - T5X;
+		    T6v = T6t - T6u;
+		    T77 = T6t + T6u;
+		    {
+			 E T6y, T6B, T6W, T6X;
+			 T6y = FMA(KP773010453, T6w, KP634393284 * T6x);
+			 T6B = FNMS(KP634393284, T6A, KP773010453 * T6z);
+			 T6C = T6y - T6B;
+			 T7h = T6B + T6y;
+			 T6W = T2x - T2o;
+			 T6X = T5V - T5S;
+			 T6Y = T6W - T6X;
+			 T7i = T6X + T6W;
+		    }
+		    T6T = FNMS(KP634393284, T6w, KP773010453 * T6x);
+		    T6U = FMA(KP634393284, T6z, KP773010453 * T6A);
+		    T6V = T6T - T6U;
+		    T78 = T6U + T6T;
+		    {
+			 E T6N, T7d, T6Q, T7e, T6M, T6O;
+			 T6M = T5I - T5H;
+			 T6N = T6L - T6M;
+			 T7d = T6L + T6M;
+			 T6O = T5v - T5e;
+			 T6Q = T6O - T6P;
+			 T7e = T6P + T6O;
+			 T6R = FNMS(KP427555093, T6Q, KP903989293 * T6N);
+			 T7n = FMA(KP941544065, T7e, KP336889853 * T7d);
+			 T73 = FMA(KP903989293, T6Q, KP427555093 * T6N);
+			 T7f = FNMS(KP336889853, T7e, KP941544065 * T7d);
+		    }
+		    {
+			 E T6G, T7a, T6J, T7b, T6E, T6I;
+			 T6E = T4g - T3Z;
+			 T6G = T6E - T6F;
+			 T7a = T6F + T6E;
+			 T6I = T4t - T4s;
+			 T6J = T6H - T6I;
+			 T7b = T6H + T6I;
+			 T6K = FMA(KP427555093, T6G, KP903989293 * T6J);
+			 T7m = FNMS(KP336889853, T7b, KP941544065 * T7a);
+			 T72 = FNMS(KP427555093, T6J, KP903989293 * T6G);
+			 T7c = FMA(KP336889853, T7a, KP941544065 * T7b);
+		    }
+	       }
+	       {
+		    E T6D, T6S, T71, T74;
+		    T6D = T6v + T6C;
+		    T6S = T6K + T6R;
+		    ro[WS(ros, 55)] = T6D - T6S;
+		    ro[WS(ros, 9)] = T6D + T6S;
+		    T71 = T6Y + T6V;
+		    T74 = T72 + T73;
+		    io[WS(ios, 9)] = T71 + T74;
+		    io[WS(ios, 55)] = T74 - T71;
+	       }
+	       {
+		    E T6Z, T70, T75, T76;
+		    T6Z = T6V - T6Y;
+		    T70 = T6R - T6K;
+		    io[WS(ios, 23)] = T6Z + T70;
+		    io[WS(ios, 41)] = T70 - T6Z;
+		    T75 = T6v - T6C;
+		    T76 = T73 - T72;
+		    ro[WS(ros, 41)] = T75 - T76;
+		    ro[WS(ros, 23)] = T75 + T76;
+	       }
+	       {
+		    E T79, T7g, T7l, T7o;
+		    T79 = T77 + T78;
+		    T7g = T7c + T7f;
+		    ro[WS(ros, 57)] = T79 - T7g;
+		    ro[WS(ros, 7)] = T79 + T7g;
+		    T7l = T7i + T7h;
+		    T7o = T7m + T7n;
+		    io[WS(ios, 7)] = T7l + T7o;
+		    io[WS(ios, 57)] = T7o - T7l;
+	       }
+	       {
+		    E T7j, T7k, T7p, T7q;
+		    T7j = T7h - T7i;
+		    T7k = T7f - T7c;
+		    io[WS(ios, 25)] = T7j + T7k;
+		    io[WS(ios, 39)] = T7k - T7j;
+		    T7p = T77 - T78;
+		    T7q = T7n - T7m;
+		    ro[WS(ros, 39)] = T7p - T7q;
+		    ro[WS(ros, 25)] = T7p + T7q;
+	       }
+	  }
+	  {
+	       E T99, T9L, T9g, T9V, T9C, T9W, T9z, T9M, T9v, Ta1, T9H, T9T, T9o, Ta0, T9G;
+	       E T9Q;
+	       {
+		    E T97, T98, T9x, T9y;
+		    T97 = T7r - T7s;
+		    T98 = T8C - T8B;
+		    T99 = T97 - T98;
+		    T9L = T97 + T98;
+		    {
+			 E T9c, T9f, T9A, T9B;
+			 T9c = FMA(KP471396736, T9a, KP881921264 * T9b);
+			 T9f = FNMS(KP471396736, T9e, KP881921264 * T9d);
+			 T9g = T9c - T9f;
+			 T9V = T9f + T9c;
+			 T9A = T7z - T7w;
+			 T9B = T8z - T8y;
+			 T9C = T9A - T9B;
+			 T9W = T9B + T9A;
+		    }
+		    T9x = FNMS(KP471396736, T9b, KP881921264 * T9a);
+		    T9y = FMA(KP881921264, T9e, KP471396736 * T9d);
+		    T9z = T9x - T9y;
+		    T9M = T9y + T9x;
+		    {
+			 E T9r, T9R, T9u, T9S, T9q, T9s;
+			 T9q = T8q - T8p;
+			 T9r = T9p - T9q;
+			 T9R = T9p + T9q;
+			 T9s = T8j - T8g;
+			 T9u = T9s - T9t;
+			 T9S = T9t + T9s;
+			 T9v = FNMS(KP514102744, T9u, KP857728610 * T9r);
+			 Ta1 = FMA(KP970031253, T9S, KP242980179 * T9R);
+			 T9H = FMA(KP857728610, T9u, KP514102744 * T9r);
+			 T9T = FNMS(KP242980179, T9S, KP970031253 * T9R);
+		    }
+		    {
+			 E T9k, T9O, T9n, T9P, T9i, T9m;
+			 T9i = T80 - T7X;
+			 T9k = T9i - T9j;
+			 T9O = T9j + T9i;
+			 T9m = T87 - T86;
+			 T9n = T9l - T9m;
+			 T9P = T9l + T9m;
+			 T9o = FMA(KP514102744, T9k, KP857728610 * T9n);
+			 Ta0 = FNMS(KP242980179, T9P, KP970031253 * T9O);
+			 T9G = FNMS(KP514102744, T9n, KP857728610 * T9k);
+			 T9Q = FMA(KP242980179, T9O, KP970031253 * T9P);
+		    }
+	       }
+	       {
+		    E T9h, T9w, T9F, T9I;
+		    T9h = T99 + T9g;
+		    T9w = T9o + T9v;
+		    ro[WS(ros, 53)] = T9h - T9w;
+		    ro[WS(ros, 11)] = T9h + T9w;
+		    T9F = T9C + T9z;
+		    T9I = T9G + T9H;
+		    io[WS(ios, 11)] = T9F + T9I;
+		    io[WS(ios, 53)] = T9I - T9F;
+	       }
+	       {
+		    E T9D, T9E, T9J, T9K;
+		    T9D = T9z - T9C;
+		    T9E = T9v - T9o;
+		    io[WS(ios, 21)] = T9D + T9E;
+		    io[WS(ios, 43)] = T9E - T9D;
+		    T9J = T99 - T9g;
+		    T9K = T9H - T9G;
+		    ro[WS(ros, 43)] = T9J - T9K;
+		    ro[WS(ros, 21)] = T9J + T9K;
+	       }
+	       {
+		    E T9N, T9U, T9Z, Ta2;
+		    T9N = T9L + T9M;
+		    T9U = T9Q + T9T;
+		    ro[WS(ros, 59)] = T9N - T9U;
+		    ro[WS(ros, 5)] = T9N + T9U;
+		    T9Z = T9W + T9V;
+		    Ta2 = Ta0 + Ta1;
+		    io[WS(ios, 5)] = T9Z + Ta2;
+		    io[WS(ios, 59)] = Ta2 - T9Z;
+	       }
+	       {
+		    E T9X, T9Y, Ta3, Ta4;
+		    T9X = T9V - T9W;
+		    T9Y = T9T - T9Q;
+		    io[WS(ios, 27)] = T9X + T9Y;
+		    io[WS(ios, 37)] = T9Y - T9X;
+		    Ta3 = T9L - T9M;
+		    Ta4 = Ta1 - Ta0;
+		    ro[WS(ros, 37)] = Ta3 - Ta4;
+		    ro[WS(ros, 27)] = Ta3 + Ta4;
+	       }
+	  }
+	  {
+	       E T2z, T69, T3g, T6j, T60, T6k, T5P, T6a, T5L, T6p, T65, T6h, T4w, T6o, T64;
+	       E T6e;
+	       {
+		    E T2f, T2y, T5N, T5O;
+		    T2f = T27 + T2e;
+		    T2y = T2o + T2x;
+		    T2z = T2f + T2y;
+		    T69 = T2f - T2y;
+		    {
+			 E T2U, T3f, T5W, T5Z;
+			 T2U = FMA(KP098017140, T2M, KP995184726 * T2T);
+			 T3f = FNMS(KP098017140, T3e, KP995184726 * T37);
+			 T3g = T2U + T3f;
+			 T6j = T3f - T2U;
+			 T5W = T5S + T5V;
+			 T5Z = T5X + T5Y;
+			 T60 = T5W + T5Z;
+			 T6k = T5Z - T5W;
+		    }
+		    T5N = FNMS(KP098017140, T2T, KP995184726 * T2M);
+		    T5O = FMA(KP995184726, T3e, KP098017140 * T37);
+		    T5P = T5N + T5O;
+		    T6a = T5O - T5N;
+		    {
+			 E T5x, T6f, T5K, T6g, T5w, T5J;
+			 T5w = T5e + T5v;
+			 T5x = T4X + T5w;
+			 T6f = T4X - T5w;
+			 T5J = T5H + T5I;
+			 T5K = T5G + T5J;
+			 T6g = T5J - T5G;
+			 T5L = FNMS(KP049067674, T5K, KP998795456 * T5x);
+			 T6p = FMA(KP671558954, T6f, KP740951125 * T6g);
+			 T65 = FMA(KP049067674, T5x, KP998795456 * T5K);
+			 T6h = FNMS(KP671558954, T6g, KP740951125 * T6f);
+		    }
+		    {
+			 E T4i, T6c, T4v, T6d, T4h, T4u;
+			 T4h = T3Z + T4g;
+			 T4i = T3I + T4h;
+			 T6c = T3I - T4h;
+			 T4u = T4s + T4t;
+			 T4v = T4r + T4u;
+			 T6d = T4u - T4r;
+			 T4w = FMA(KP998795456, T4i, KP049067674 * T4v);
+			 T6o = FNMS(KP671558954, T6c, KP740951125 * T6d);
+			 T64 = FNMS(KP049067674, T4i, KP998795456 * T4v);
+			 T6e = FMA(KP740951125, T6c, KP671558954 * T6d);
+		    }
+	       }
+	       {
+		    E T3h, T5M, T63, T66;
+		    T3h = T2z + T3g;
+		    T5M = T4w + T5L;
+		    ro[WS(ros, 63)] = T3h - T5M;
+		    ro[WS(ros, 1)] = T3h + T5M;
+		    T63 = T60 + T5P;
+		    T66 = T64 + T65;
+		    io[WS(ios, 1)] = T63 + T66;
+		    io[WS(ios, 63)] = T66 - T63;
+	       }
+	       {
+		    E T61, T62, T67, T68;
+		    T61 = T5P - T60;
+		    T62 = T5L - T4w;
+		    io[WS(ios, 31)] = T61 + T62;
+		    io[WS(ios, 33)] = T62 - T61;
+		    T67 = T2z - T3g;
+		    T68 = T65 - T64;
+		    ro[WS(ros, 33)] = T67 - T68;
+		    ro[WS(ros, 31)] = T67 + T68;
+	       }
+	       {
+		    E T6b, T6i, T6n, T6q;
+		    T6b = T69 + T6a;
+		    T6i = T6e + T6h;
+		    ro[WS(ros, 49)] = T6b - T6i;
+		    ro[WS(ros, 15)] = T6b + T6i;
+		    T6n = T6k + T6j;
+		    T6q = T6o + T6p;
+		    io[WS(ios, 15)] = T6n + T6q;
+		    io[WS(ios, 49)] = T6q - T6n;
+	       }
+	       {
+		    E T6l, T6m, T6r, T6s;
+		    T6l = T6j - T6k;
+		    T6m = T6h - T6e;
+		    io[WS(ios, 17)] = T6l + T6m;
+		    io[WS(ios, 47)] = T6m - T6l;
+		    T6r = T69 - T6a;
+		    T6s = T6p - T6o;
+		    ro[WS(ros, 47)] = T6r - T6s;
+		    ro[WS(ros, 17)] = T6r + T6s;
+	       }
+	  }
+	  {
+	       E T7B, T8N, T7Q, T8X, T8E, T8Y, T8x, T8O, T8t, T93, T8J, T8V, T8a, T92, T8I;
+	       E T8S;
+	       {
+		    E T7t, T7A, T8v, T8w;
+		    T7t = T7r + T7s;
+		    T7A = T7w + T7z;
+		    T7B = T7t + T7A;
+		    T8N = T7t - T7A;
+		    {
+			 E T7I, T7P, T8A, T8D;
+			 T7I = FMA(KP956940335, T7E, KP290284677 * T7H);
+			 T7P = FNMS(KP290284677, T7O, KP956940335 * T7L);
+			 T7Q = T7I + T7P;
+			 T8X = T7P - T7I;
+			 T8A = T8y + T8z;
+			 T8D = T8B + T8C;
+			 T8E = T8A + T8D;
+			 T8Y = T8D - T8A;
+		    }
+		    T8v = FNMS(KP290284677, T7E, KP956940335 * T7H);
+		    T8w = FMA(KP290284677, T7L, KP956940335 * T7O);
+		    T8x = T8v + T8w;
+		    T8O = T8w - T8v;
+		    {
+			 E T8l, T8T, T8s, T8U, T8k, T8r;
+			 T8k = T8g + T8j;
+			 T8l = T8d + T8k;
+			 T8T = T8d - T8k;
+			 T8r = T8p + T8q;
+			 T8s = T8o + T8r;
+			 T8U = T8r - T8o;
+			 T8t = FNMS(KP146730474, T8s, KP989176509 * T8l);
+			 T93 = FMA(KP595699304, T8T, KP803207531 * T8U);
+			 T8J = FMA(KP146730474, T8l, KP989176509 * T8s);
+			 T8V = FNMS(KP595699304, T8U, KP803207531 * T8T);
+		    }
+		    {
+			 E T82, T8Q, T89, T8R, T81, T88;
+			 T81 = T7X + T80;
+			 T82 = T7U + T81;
+			 T8Q = T7U - T81;
+			 T88 = T86 + T87;
+			 T89 = T85 + T88;
+			 T8R = T88 - T85;
+			 T8a = FMA(KP989176509, T82, KP146730474 * T89);
+			 T92 = FNMS(KP595699304, T8Q, KP803207531 * T8R);
+			 T8I = FNMS(KP146730474, T82, KP989176509 * T89);
+			 T8S = FMA(KP803207531, T8Q, KP595699304 * T8R);
+		    }
+	       }
+	       {
+		    E T7R, T8u, T8H, T8K;
+		    T7R = T7B + T7Q;
+		    T8u = T8a + T8t;
+		    ro[WS(ros, 61)] = T7R - T8u;
+		    ro[WS(ros, 3)] = T7R + T8u;
+		    T8H = T8E + T8x;
+		    T8K = T8I + T8J;
+		    io[WS(ios, 3)] = T8H + T8K;
+		    io[WS(ios, 61)] = T8K - T8H;
+	       }
+	       {
+		    E T8F, T8G, T8L, T8M;
+		    T8F = T8x - T8E;
+		    T8G = T8t - T8a;
+		    io[WS(ios, 29)] = T8F + T8G;
+		    io[WS(ios, 35)] = T8G - T8F;
+		    T8L = T7B - T7Q;
+		    T8M = T8J - T8I;
+		    ro[WS(ros, 35)] = T8L - T8M;
+		    ro[WS(ros, 29)] = T8L + T8M;
+	       }
+	       {
+		    E T8P, T8W, T91, T94;
+		    T8P = T8N + T8O;
+		    T8W = T8S + T8V;
+		    ro[WS(ros, 51)] = T8P - T8W;
+		    ro[WS(ros, 13)] = T8P + T8W;
+		    T91 = T8Y + T8X;
+		    T94 = T92 + T93;
+		    io[WS(ios, 13)] = T91 + T94;
+		    io[WS(ios, 51)] = T94 - T91;
+	       }
+	       {
+		    E T8Z, T90, T95, T96;
+		    T8Z = T8X - T8Y;
+		    T90 = T8V - T8S;
+		    io[WS(ios, 19)] = T8Z + T90;
+		    io[WS(ios, 45)] = T90 - T8Z;
+		    T95 = T8N - T8O;
+		    T96 = T93 - T92;
+		    ro[WS(ros, 45)] = T95 - T96;
+		    ro[WS(ros, 19)] = T95 + T96;
+	       }
+	  }
+     }
+}
+
+static void mr2hc_128(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  mr2hc_128_0(I, ro, io, is, ros, ios);
+	  I += ivs;
+	  ro += ovs;
+	  io += ovs;
+     }
+}
+
+static const kr2hc_desc desc = { 128, "mr2hc_128", {812, 186, 144, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_mr2hc_128) (planner *p) {
+     X(kr2hc_register) (p, mr2hc_128, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/mr2hc_32.c b/src/fftw3/rdft/codelets/r2hc/mr2hc_32.c
new file mode 100644
index 0000000..4f351a4
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/mr2hc_32.c
@@ -0,0 +1,330 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:45 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc_noinline -compact -variables 4 -n 32 -name mr2hc_32 -include r2hc.h */
+
+/*
+ * This function contains 156 FP additions, 42 FP multiplications,
+ * (or, 140 additions, 26 multiplications, 16 fused multiply/add),
+ * 53 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: mr2hc_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: mr2hc_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: mr2hc_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void mr2hc_32_0(const R *I, R *ro, R *io, stride is, stride ros, stride ios)
+{
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     {
+	  E T7, T2b, Tv, T1l, Te, T2o, Ty, T1k, Tt, T2d, TF, T1h, Tm, T2c, TC;
+	  E T1i, T1Z, T22, T2k, T2j, T1e, T1C, T19, T1B, T1S, T1V, T2h, T2g, TX, T1z;
+	  E TS, T1y;
+	  {
+	       E T1, T2, T3, T4, T5, T6;
+	       T1 = I[0];
+	       T2 = I[WS(is, 16)];
+	       T3 = T1 + T2;
+	       T4 = I[WS(is, 8)];
+	       T5 = I[WS(is, 24)];
+	       T6 = T4 + T5;
+	       T7 = T3 + T6;
+	       T2b = T3 - T6;
+	       Tv = T1 - T2;
+	       T1l = T4 - T5;
+	  }
+	  {
+	       E Ta, Tw, Td, Tx;
+	       {
+		    E T8, T9, Tb, Tc;
+		    T8 = I[WS(is, 4)];
+		    T9 = I[WS(is, 20)];
+		    Ta = T8 + T9;
+		    Tw = T8 - T9;
+		    Tb = I[WS(is, 28)];
+		    Tc = I[WS(is, 12)];
+		    Td = Tb + Tc;
+		    Tx = Tb - Tc;
+	       }
+	       Te = Ta + Td;
+	       T2o = Td - Ta;
+	       Ty = KP707106781 * (Tw + Tx);
+	       T1k = KP707106781 * (Tx - Tw);
+	  }
+	  {
+	       E Tp, TD, Ts, TE;
+	       {
+		    E Tn, To, Tq, Tr;
+		    Tn = I[WS(is, 30)];
+		    To = I[WS(is, 14)];
+		    Tp = Tn + To;
+		    TD = Tn - To;
+		    Tq = I[WS(is, 6)];
+		    Tr = I[WS(is, 22)];
+		    Ts = Tq + Tr;
+		    TE = Tq - Tr;
+	       }
+	       Tt = Tp + Ts;
+	       T2d = Tp - Ts;
+	       TF = FMA(KP923879532, TD, KP382683432 * TE);
+	       T1h = FNMS(KP923879532, TE, KP382683432 * TD);
+	  }
+	  {
+	       E Ti, TA, Tl, TB;
+	       {
+		    E Tg, Th, Tj, Tk;
+		    Tg = I[WS(is, 2)];
+		    Th = I[WS(is, 18)];
+		    Ti = Tg + Th;
+		    TA = Tg - Th;
+		    Tj = I[WS(is, 10)];
+		    Tk = I[WS(is, 26)];
+		    Tl = Tj + Tk;
+		    TB = Tj - Tk;
+	       }
+	       Tm = Ti + Tl;
+	       T2c = Ti - Tl;
+	       TC = FNMS(KP382683432, TB, KP923879532 * TA);
+	       T1i = FMA(KP382683432, TA, KP923879532 * TB);
+	  }
+	  {
+	       E T11, T1X, T1d, T1Y, T14, T20, T17, T21, T1a, T18;
+	       {
+		    E TZ, T10, T1b, T1c;
+		    TZ = I[WS(is, 31)];
+		    T10 = I[WS(is, 15)];
+		    T11 = TZ - T10;
+		    T1X = TZ + T10;
+		    T1b = I[WS(is, 7)];
+		    T1c = I[WS(is, 23)];
+		    T1d = T1b - T1c;
+		    T1Y = T1b + T1c;
+	       }
+	       {
+		    E T12, T13, T15, T16;
+		    T12 = I[WS(is, 3)];
+		    T13 = I[WS(is, 19)];
+		    T14 = T12 - T13;
+		    T20 = T12 + T13;
+		    T15 = I[WS(is, 27)];
+		    T16 = I[WS(is, 11)];
+		    T17 = T15 - T16;
+		    T21 = T15 + T16;
+	       }
+	       T1Z = T1X + T1Y;
+	       T22 = T20 + T21;
+	       T2k = T21 - T20;
+	       T2j = T1X - T1Y;
+	       T1a = KP707106781 * (T17 - T14);
+	       T1e = T1a - T1d;
+	       T1C = T1d + T1a;
+	       T18 = KP707106781 * (T14 + T17);
+	       T19 = T11 + T18;
+	       T1B = T11 - T18;
+	  }
+	  {
+	       E TK, T1Q, TW, T1R, TN, T1T, TQ, T1U, TT, TR;
+	       {
+		    E TI, TJ, TU, TV;
+		    TI = I[WS(is, 1)];
+		    TJ = I[WS(is, 17)];
+		    TK = TI - TJ;
+		    T1Q = TI + TJ;
+		    TU = I[WS(is, 9)];
+		    TV = I[WS(is, 25)];
+		    TW = TU - TV;
+		    T1R = TU + TV;
+	       }
+	       {
+		    E TL, TM, TO, TP;
+		    TL = I[WS(is, 5)];
+		    TM = I[WS(is, 21)];
+		    TN = TL - TM;
+		    T1T = TL + TM;
+		    TO = I[WS(is, 29)];
+		    TP = I[WS(is, 13)];
+		    TQ = TO - TP;
+		    T1U = TO + TP;
+	       }
+	       T1S = T1Q + T1R;
+	       T1V = T1T + T1U;
+	       T2h = T1U - T1T;
+	       T2g = T1Q - T1R;
+	       TT = KP707106781 * (TQ - TN);
+	       TX = TT - TW;
+	       T1z = TW + TT;
+	       TR = KP707106781 * (TN + TQ);
+	       TS = TK + TR;
+	       T1y = TK - TR;
+	  }
+	  {
+	       E Tf, Tu, T27, T28, T29, T2a;
+	       Tf = T7 + Te;
+	       Tu = Tm + Tt;
+	       T27 = Tf + Tu;
+	       T28 = T1S + T1V;
+	       T29 = T1Z + T22;
+	       T2a = T28 + T29;
+	       ro[WS(ros, 8)] = Tf - Tu;
+	       io[WS(ios, 8)] = T29 - T28;
+	       ro[WS(ros, 16)] = T27 - T2a;
+	       ro[0] = T27 + T2a;
+	  }
+	  {
+	       E T1P, T25, T24, T26, T1W, T23;
+	       T1P = T7 - Te;
+	       T25 = Tt - Tm;
+	       T1W = T1S - T1V;
+	       T23 = T1Z - T22;
+	       T24 = KP707106781 * (T1W + T23);
+	       T26 = KP707106781 * (T23 - T1W);
+	       ro[WS(ros, 12)] = T1P - T24;
+	       io[WS(ios, 12)] = T26 - T25;
+	       ro[WS(ros, 4)] = T1P + T24;
+	       io[WS(ios, 4)] = T25 + T26;
+	  }
+	  {
+	       E T2f, T2v, T2p, T2r, T2m, T2q, T2u, T2w, T2e, T2n;
+	       T2e = KP707106781 * (T2c + T2d);
+	       T2f = T2b + T2e;
+	       T2v = T2b - T2e;
+	       T2n = KP707106781 * (T2d - T2c);
+	       T2p = T2n - T2o;
+	       T2r = T2o + T2n;
+	       {
+		    E T2i, T2l, T2s, T2t;
+		    T2i = FMA(KP923879532, T2g, KP382683432 * T2h);
+		    T2l = FNMS(KP382683432, T2k, KP923879532 * T2j);
+		    T2m = T2i + T2l;
+		    T2q = T2l - T2i;
+		    T2s = FNMS(KP382683432, T2g, KP923879532 * T2h);
+		    T2t = FMA(KP382683432, T2j, KP923879532 * T2k);
+		    T2u = T2s + T2t;
+		    T2w = T2t - T2s;
+	       }
+	       ro[WS(ros, 14)] = T2f - T2m;
+	       io[WS(ios, 14)] = T2u - T2r;
+	       ro[WS(ros, 2)] = T2f + T2m;
+	       io[WS(ios, 2)] = T2r + T2u;
+	       io[WS(ios, 6)] = T2p + T2q;
+	       ro[WS(ros, 6)] = T2v + T2w;
+	       io[WS(ios, 10)] = T2q - T2p;
+	       ro[WS(ros, 10)] = T2v - T2w;
+	  }
+	  {
+	       E TH, T1t, T1s, T1u, T1g, T1o, T1n, T1p;
+	       {
+		    E Tz, TG, T1q, T1r;
+		    Tz = Tv + Ty;
+		    TG = TC + TF;
+		    TH = Tz + TG;
+		    T1t = Tz - TG;
+		    T1q = FNMS(KP195090322, TS, KP980785280 * TX);
+		    T1r = FMA(KP195090322, T19, KP980785280 * T1e);
+		    T1s = T1q + T1r;
+		    T1u = T1r - T1q;
+	       }
+	       {
+		    E TY, T1f, T1j, T1m;
+		    TY = FMA(KP980785280, TS, KP195090322 * TX);
+		    T1f = FNMS(KP195090322, T1e, KP980785280 * T19);
+		    T1g = TY + T1f;
+		    T1o = T1f - TY;
+		    T1j = T1h - T1i;
+		    T1m = T1k - T1l;
+		    T1n = T1j - T1m;
+		    T1p = T1m + T1j;
+	       }
+	       ro[WS(ros, 15)] = TH - T1g;
+	       io[WS(ios, 15)] = T1s - T1p;
+	       ro[WS(ros, 1)] = TH + T1g;
+	       io[WS(ios, 1)] = T1p + T1s;
+	       io[WS(ios, 7)] = T1n + T1o;
+	       ro[WS(ros, 7)] = T1t + T1u;
+	       io[WS(ios, 9)] = T1o - T1n;
+	       ro[WS(ros, 9)] = T1t - T1u;
+	  }
+	  {
+	       E T1x, T1N, T1M, T1O, T1E, T1I, T1H, T1J;
+	       {
+		    E T1v, T1w, T1K, T1L;
+		    T1v = Tv - Ty;
+		    T1w = T1i + T1h;
+		    T1x = T1v + T1w;
+		    T1N = T1v - T1w;
+		    T1K = FNMS(KP555570233, T1y, KP831469612 * T1z);
+		    T1L = FMA(KP555570233, T1B, KP831469612 * T1C);
+		    T1M = T1K + T1L;
+		    T1O = T1L - T1K;
+	       }
+	       {
+		    E T1A, T1D, T1F, T1G;
+		    T1A = FMA(KP831469612, T1y, KP555570233 * T1z);
+		    T1D = FNMS(KP555570233, T1C, KP831469612 * T1B);
+		    T1E = T1A + T1D;
+		    T1I = T1D - T1A;
+		    T1F = TF - TC;
+		    T1G = T1l + T1k;
+		    T1H = T1F - T1G;
+		    T1J = T1G + T1F;
+	       }
+	       ro[WS(ros, 13)] = T1x - T1E;
+	       io[WS(ios, 13)] = T1M - T1J;
+	       ro[WS(ros, 3)] = T1x + T1E;
+	       io[WS(ios, 3)] = T1J + T1M;
+	       io[WS(ios, 5)] = T1H + T1I;
+	       ro[WS(ros, 5)] = T1N + T1O;
+	       io[WS(ios, 11)] = T1I - T1H;
+	       ro[WS(ros, 11)] = T1N - T1O;
+	  }
+     }
+}
+
+static void mr2hc_32(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  mr2hc_32_0(I, ro, io, is, ros, ios);
+	  I += ivs;
+	  ro += ovs;
+	  io += ovs;
+     }
+}
+
+static const kr2hc_desc desc = { 32, "mr2hc_32", {140, 26, 16, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_mr2hc_32) (planner *p) {
+     X(kr2hc_register) (p, mr2hc_32, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/mr2hc_64.c b/src/fftw3/rdft/codelets/r2hc/mr2hc_64.c
new file mode 100644
index 0000000..c859fbf
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/mr2hc_64.c
@@ -0,0 +1,729 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:49 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc_noinline -compact -variables 4 -n 64 -name mr2hc_64 -include r2hc.h */
+
+/*
+ * This function contains 394 FP additions, 124 FP multiplications,
+ * (or, 342 additions, 72 multiplications, 52 fused multiply/add),
+ * 105 stack variables, and 128 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: mr2hc_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: mr2hc_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: mr2hc_64.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void mr2hc_64_0(const R *I, R *ro, R *io, stride is, stride ros, stride ios)
+{
+     DK(KP773010453, +0.773010453362736960810906609758469800971041293);
+     DK(KP634393284, +0.634393284163645498215171613225493370675687095);
+     DK(KP098017140, +0.098017140329560601994195563888641845861136673);
+     DK(KP995184726, +0.995184726672196886244836953109479921575474869);
+     DK(KP290284677, +0.290284677254462367636192375817395274691476278);
+     DK(KP956940335, +0.956940335732208864935797886980269969482849206);
+     DK(KP471396736, +0.471396736825997648556387625905254377657460319);
+     DK(KP881921264, +0.881921264348355029712756863660388349508442621);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     {
+	  E T4l, T5a, T15, T3n, T2T, T3Q, T7, Te, Tf, T4A, T4L, T1X, T3B, T23, T3y;
+	  E T5I, T66, T4R, T52, T2j, T3F, T2H, T3I, T5P, T69, T1i, T3t, T1l, T3u, TZ;
+	  E T63, T4v, T58, T1r, T3r, T1u, T3q, TK, T62, T4s, T57, Tm, Tt, Tu, T4o;
+	  E T5b, T1c, T3R, T2Q, T3o, T1M, T3z, T5L, T67, T26, T3C, T4H, T4M, T2y, T3J;
+	  E T5S, T6a, T2C, T3G, T4Y, T53;
+	  {
+	       E T3, T11, Td, T13, T6, T2S, Ta, T12, T14, T2R;
+	       {
+		    E T1, T2, Tb, Tc;
+		    T1 = I[0];
+		    T2 = I[WS(is, 32)];
+		    T3 = T1 + T2;
+		    T11 = T1 - T2;
+		    Tb = I[WS(is, 56)];
+		    Tc = I[WS(is, 24)];
+		    Td = Tb + Tc;
+		    T13 = Tb - Tc;
+	       }
+	       {
+		    E T4, T5, T8, T9;
+		    T4 = I[WS(is, 16)];
+		    T5 = I[WS(is, 48)];
+		    T6 = T4 + T5;
+		    T2S = T4 - T5;
+		    T8 = I[WS(is, 8)];
+		    T9 = I[WS(is, 40)];
+		    Ta = T8 + T9;
+		    T12 = T8 - T9;
+	       }
+	       T4l = T3 - T6;
+	       T5a = Td - Ta;
+	       T14 = KP707106781 * (T12 + T13);
+	       T15 = T11 + T14;
+	       T3n = T11 - T14;
+	       T2R = KP707106781 * (T13 - T12);
+	       T2T = T2R - T2S;
+	       T3Q = T2S + T2R;
+	       T7 = T3 + T6;
+	       Te = Ta + Td;
+	       Tf = T7 + Te;
+	  }
+	  {
+	       E T1P, T4J, T21, T4y, T1S, T4K, T1W, T4z;
+	       {
+		    E T1N, T1O, T1Z, T20;
+		    T1N = I[WS(is, 57)];
+		    T1O = I[WS(is, 25)];
+		    T1P = T1N - T1O;
+		    T4J = T1N + T1O;
+		    T1Z = I[WS(is, 1)];
+		    T20 = I[WS(is, 33)];
+		    T21 = T1Z - T20;
+		    T4y = T1Z + T20;
+	       }
+	       {
+		    E T1Q, T1R, T1U, T1V;
+		    T1Q = I[WS(is, 9)];
+		    T1R = I[WS(is, 41)];
+		    T1S = T1Q - T1R;
+		    T4K = T1Q + T1R;
+		    T1U = I[WS(is, 17)];
+		    T1V = I[WS(is, 49)];
+		    T1W = T1U - T1V;
+		    T4z = T1U + T1V;
+	       }
+	       T4A = T4y - T4z;
+	       T4L = T4J - T4K;
+	       {
+		    E T1T, T22, T5G, T5H;
+		    T1T = KP707106781 * (T1P - T1S);
+		    T1X = T1T - T1W;
+		    T3B = T1W + T1T;
+		    T22 = KP707106781 * (T1S + T1P);
+		    T23 = T21 + T22;
+		    T3y = T21 - T22;
+		    T5G = T4y + T4z;
+		    T5H = T4K + T4J;
+		    T5I = T5G + T5H;
+		    T66 = T5G - T5H;
+	       }
+	  }
+	  {
+	       E T2b, T4P, T2G, T4Q, T2e, T51, T2h, T50;
+	       {
+		    E T29, T2a, T2E, T2F;
+		    T29 = I[WS(is, 63)];
+		    T2a = I[WS(is, 31)];
+		    T2b = T29 - T2a;
+		    T4P = T29 + T2a;
+		    T2E = I[WS(is, 15)];
+		    T2F = I[WS(is, 47)];
+		    T2G = T2E - T2F;
+		    T4Q = T2E + T2F;
+	       }
+	       {
+		    E T2c, T2d, T2f, T2g;
+		    T2c = I[WS(is, 7)];
+		    T2d = I[WS(is, 39)];
+		    T2e = T2c - T2d;
+		    T51 = T2c + T2d;
+		    T2f = I[WS(is, 55)];
+		    T2g = I[WS(is, 23)];
+		    T2h = T2f - T2g;
+		    T50 = T2f + T2g;
+	       }
+	       T4R = T4P - T4Q;
+	       T52 = T50 - T51;
+	       {
+		    E T2i, T2D, T5N, T5O;
+		    T2i = KP707106781 * (T2e + T2h);
+		    T2j = T2b + T2i;
+		    T3F = T2b - T2i;
+		    T2D = KP707106781 * (T2h - T2e);
+		    T2H = T2D - T2G;
+		    T3I = T2G + T2D;
+		    T5N = T4P + T4Q;
+		    T5O = T51 + T50;
+		    T5P = T5N + T5O;
+		    T69 = T5N - T5O;
+	       }
+	  }
+	  {
+	       E TN, T1e, TX, T1g, TQ, T1k, TU, T1f, T1h, T1j;
+	       {
+		    E TL, TM, TV, TW;
+		    TL = I[WS(is, 62)];
+		    TM = I[WS(is, 30)];
+		    TN = TL + TM;
+		    T1e = TL - TM;
+		    TV = I[WS(is, 54)];
+		    TW = I[WS(is, 22)];
+		    TX = TV + TW;
+		    T1g = TV - TW;
+	       }
+	       {
+		    E TO, TP, TS, TT;
+		    TO = I[WS(is, 14)];
+		    TP = I[WS(is, 46)];
+		    TQ = TO + TP;
+		    T1k = TO - TP;
+		    TS = I[WS(is, 6)];
+		    TT = I[WS(is, 38)];
+		    TU = TS + TT;
+		    T1f = TS - TT;
+	       }
+	       T1h = KP707106781 * (T1f + T1g);
+	       T1i = T1e + T1h;
+	       T3t = T1e - T1h;
+	       T1j = KP707106781 * (T1g - T1f);
+	       T1l = T1j - T1k;
+	       T3u = T1k + T1j;
+	       {
+		    E TR, TY, T4t, T4u;
+		    TR = TN + TQ;
+		    TY = TU + TX;
+		    TZ = TR + TY;
+		    T63 = TR - TY;
+		    T4t = TN - TQ;
+		    T4u = TX - TU;
+		    T4v = FNMS(KP382683432, T4u, KP923879532 * T4t);
+		    T58 = FMA(KP382683432, T4t, KP923879532 * T4u);
+	       }
+	  }
+	  {
+	       E Ty, T1s, TI, T1n, TB, T1q, TF, T1o, T1p, T1t;
+	       {
+		    E Tw, Tx, TG, TH;
+		    Tw = I[WS(is, 2)];
+		    Tx = I[WS(is, 34)];
+		    Ty = Tw + Tx;
+		    T1s = Tw - Tx;
+		    TG = I[WS(is, 58)];
+		    TH = I[WS(is, 26)];
+		    TI = TG + TH;
+		    T1n = TG - TH;
+	       }
+	       {
+		    E Tz, TA, TD, TE;
+		    Tz = I[WS(is, 18)];
+		    TA = I[WS(is, 50)];
+		    TB = Tz + TA;
+		    T1q = Tz - TA;
+		    TD = I[WS(is, 10)];
+		    TE = I[WS(is, 42)];
+		    TF = TD + TE;
+		    T1o = TD - TE;
+	       }
+	       T1p = KP707106781 * (T1n - T1o);
+	       T1r = T1p - T1q;
+	       T3r = T1q + T1p;
+	       T1t = KP707106781 * (T1o + T1n);
+	       T1u = T1s + T1t;
+	       T3q = T1s - T1t;
+	       {
+		    E TC, TJ, T4q, T4r;
+		    TC = Ty + TB;
+		    TJ = TF + TI;
+		    TK = TC + TJ;
+		    T62 = TC - TJ;
+		    T4q = Ty - TB;
+		    T4r = TI - TF;
+		    T4s = FMA(KP923879532, T4q, KP382683432 * T4r);
+		    T57 = FNMS(KP382683432, T4q, KP923879532 * T4r);
+	       }
+	  }
+	  {
+	       E Ti, T16, Ts, T1a, Tl, T17, Tp, T19, T4m, T4n;
+	       {
+		    E Tg, Th, Tq, Tr;
+		    Tg = I[WS(is, 4)];
+		    Th = I[WS(is, 36)];
+		    Ti = Tg + Th;
+		    T16 = Tg - Th;
+		    Tq = I[WS(is, 12)];
+		    Tr = I[WS(is, 44)];
+		    Ts = Tq + Tr;
+		    T1a = Tq - Tr;
+	       }
+	       {
+		    E Tj, Tk, Tn, To;
+		    Tj = I[WS(is, 20)];
+		    Tk = I[WS(is, 52)];
+		    Tl = Tj + Tk;
+		    T17 = Tj - Tk;
+		    Tn = I[WS(is, 60)];
+		    To = I[WS(is, 28)];
+		    Tp = Tn + To;
+		    T19 = Tn - To;
+	       }
+	       Tm = Ti + Tl;
+	       Tt = Tp + Ts;
+	       Tu = Tm + Tt;
+	       T4m = Ti - Tl;
+	       T4n = Tp - Ts;
+	       T4o = KP707106781 * (T4m + T4n);
+	       T5b = KP707106781 * (T4n - T4m);
+	       {
+		    E T18, T1b, T2O, T2P;
+		    T18 = FNMS(KP382683432, T17, KP923879532 * T16);
+		    T1b = FMA(KP923879532, T19, KP382683432 * T1a);
+		    T1c = T18 + T1b;
+		    T3R = T1b - T18;
+		    T2O = FNMS(KP923879532, T1a, KP382683432 * T19);
+		    T2P = FMA(KP382683432, T16, KP923879532 * T17);
+		    T2Q = T2O - T2P;
+		    T3o = T2P + T2O;
+	       }
+	  }
+	  {
+	       E T1A, T4E, T1K, T4C, T1D, T4F, T1H, T4B;
+	       {
+		    E T1y, T1z, T1I, T1J;
+		    T1y = I[WS(is, 61)];
+		    T1z = I[WS(is, 29)];
+		    T1A = T1y - T1z;
+		    T4E = T1y + T1z;
+		    T1I = I[WS(is, 21)];
+		    T1J = I[WS(is, 53)];
+		    T1K = T1I - T1J;
+		    T4C = T1I + T1J;
+	       }
+	       {
+		    E T1B, T1C, T1F, T1G;
+		    T1B = I[WS(is, 13)];
+		    T1C = I[WS(is, 45)];
+		    T1D = T1B - T1C;
+		    T4F = T1B + T1C;
+		    T1F = I[WS(is, 5)];
+		    T1G = I[WS(is, 37)];
+		    T1H = T1F - T1G;
+		    T4B = T1F + T1G;
+	       }
+	       {
+		    E T1E, T1L, T5J, T5K;
+		    T1E = FNMS(KP923879532, T1D, KP382683432 * T1A);
+		    T1L = FMA(KP382683432, T1H, KP923879532 * T1K);
+		    T1M = T1E - T1L;
+		    T3z = T1L + T1E;
+		    T5J = T4B + T4C;
+		    T5K = T4E + T4F;
+		    T5L = T5J + T5K;
+		    T67 = T5K - T5J;
+	       }
+	       {
+		    E T24, T25, T4D, T4G;
+		    T24 = FNMS(KP382683432, T1K, KP923879532 * T1H);
+		    T25 = FMA(KP923879532, T1A, KP382683432 * T1D);
+		    T26 = T24 + T25;
+		    T3C = T25 - T24;
+		    T4D = T4B - T4C;
+		    T4G = T4E - T4F;
+		    T4H = KP707106781 * (T4D + T4G);
+		    T4M = KP707106781 * (T4G - T4D);
+	       }
+	  }
+	  {
+	       E T2m, T4S, T2w, T4W, T2p, T4T, T2t, T4V;
+	       {
+		    E T2k, T2l, T2u, T2v;
+		    T2k = I[WS(is, 3)];
+		    T2l = I[WS(is, 35)];
+		    T2m = T2k - T2l;
+		    T4S = T2k + T2l;
+		    T2u = I[WS(is, 11)];
+		    T2v = I[WS(is, 43)];
+		    T2w = T2u - T2v;
+		    T4W = T2u + T2v;
+	       }
+	       {
+		    E T2n, T2o, T2r, T2s;
+		    T2n = I[WS(is, 19)];
+		    T2o = I[WS(is, 51)];
+		    T2p = T2n - T2o;
+		    T4T = T2n + T2o;
+		    T2r = I[WS(is, 59)];
+		    T2s = I[WS(is, 27)];
+		    T2t = T2r - T2s;
+		    T4V = T2r + T2s;
+	       }
+	       {
+		    E T2q, T2x, T5Q, T5R;
+		    T2q = FNMS(KP382683432, T2p, KP923879532 * T2m);
+		    T2x = FMA(KP923879532, T2t, KP382683432 * T2w);
+		    T2y = T2q + T2x;
+		    T3J = T2x - T2q;
+		    T5Q = T4S + T4T;
+		    T5R = T4V + T4W;
+		    T5S = T5Q + T5R;
+		    T6a = T5R - T5Q;
+	       }
+	       {
+		    E T2A, T2B, T4U, T4X;
+		    T2A = FNMS(KP923879532, T2w, KP382683432 * T2t);
+		    T2B = FMA(KP382683432, T2m, KP923879532 * T2p);
+		    T2C = T2A - T2B;
+		    T3G = T2B + T2A;
+		    T4U = T4S - T4T;
+		    T4X = T4V - T4W;
+		    T4Y = KP707106781 * (T4U + T4X);
+		    T53 = KP707106781 * (T4X - T4U);
+	       }
+	  }
+	  {
+	       E Tv, T10, T5X, T5Y, T5Z, T60;
+	       Tv = Tf + Tu;
+	       T10 = TK + TZ;
+	       T5X = Tv + T10;
+	       T5Y = T5I + T5L;
+	       T5Z = T5P + T5S;
+	       T60 = T5Y + T5Z;
+	       ro[WS(ros, 16)] = Tv - T10;
+	       io[WS(ios, 16)] = T5Z - T5Y;
+	       ro[WS(ros, 32)] = T5X - T60;
+	       ro[0] = T5X + T60;
+	  }
+	  {
+	       E T5F, T5V, T5U, T5W, T5M, T5T;
+	       T5F = Tf - Tu;
+	       T5V = TZ - TK;
+	       T5M = T5I - T5L;
+	       T5T = T5P - T5S;
+	       T5U = KP707106781 * (T5M + T5T);
+	       T5W = KP707106781 * (T5T - T5M);
+	       ro[WS(ros, 24)] = T5F - T5U;
+	       io[WS(ios, 24)] = T5W - T5V;
+	       ro[WS(ros, 8)] = T5F + T5U;
+	       io[WS(ios, 8)] = T5V + T5W;
+	  }
+	  {
+	       E T65, T6l, T6k, T6m, T6c, T6g, T6f, T6h;
+	       {
+		    E T61, T64, T6i, T6j;
+		    T61 = T7 - Te;
+		    T64 = KP707106781 * (T62 + T63);
+		    T65 = T61 + T64;
+		    T6l = T61 - T64;
+		    T6i = FNMS(KP382683432, T66, KP923879532 * T67);
+		    T6j = FMA(KP382683432, T69, KP923879532 * T6a);
+		    T6k = T6i + T6j;
+		    T6m = T6j - T6i;
+	       }
+	       {
+		    E T68, T6b, T6d, T6e;
+		    T68 = FMA(KP923879532, T66, KP382683432 * T67);
+		    T6b = FNMS(KP382683432, T6a, KP923879532 * T69);
+		    T6c = T68 + T6b;
+		    T6g = T6b - T68;
+		    T6d = KP707106781 * (T63 - T62);
+		    T6e = Tt - Tm;
+		    T6f = T6d - T6e;
+		    T6h = T6e + T6d;
+	       }
+	       ro[WS(ros, 28)] = T65 - T6c;
+	       io[WS(ios, 28)] = T6k - T6h;
+	       ro[WS(ros, 4)] = T65 + T6c;
+	       io[WS(ios, 4)] = T6h + T6k;
+	       io[WS(ios, 12)] = T6f + T6g;
+	       ro[WS(ros, 12)] = T6l + T6m;
+	       io[WS(ios, 20)] = T6g - T6f;
+	       ro[WS(ros, 20)] = T6l - T6m;
+	  }
+	  {
+	       E T5n, T5D, T5x, T5z, T5q, T5A, T5t, T5B;
+	       {
+		    E T5l, T5m, T5v, T5w;
+		    T5l = T4l - T4o;
+		    T5m = T58 - T57;
+		    T5n = T5l + T5m;
+		    T5D = T5l - T5m;
+		    T5v = T4v - T4s;
+		    T5w = T5b - T5a;
+		    T5x = T5v - T5w;
+		    T5z = T5w + T5v;
+	       }
+	       {
+		    E T5o, T5p, T5r, T5s;
+		    T5o = T4A - T4H;
+		    T5p = T4M - T4L;
+		    T5q = FMA(KP831469612, T5o, KP555570233 * T5p);
+		    T5A = FNMS(KP555570233, T5o, KP831469612 * T5p);
+		    T5r = T4R - T4Y;
+		    T5s = T53 - T52;
+		    T5t = FNMS(KP555570233, T5s, KP831469612 * T5r);
+		    T5B = FMA(KP555570233, T5r, KP831469612 * T5s);
+	       }
+	       {
+		    E T5u, T5C, T5y, T5E;
+		    T5u = T5q + T5t;
+		    ro[WS(ros, 26)] = T5n - T5u;
+		    ro[WS(ros, 6)] = T5n + T5u;
+		    T5C = T5A + T5B;
+		    io[WS(ios, 6)] = T5z + T5C;
+		    io[WS(ios, 26)] = T5C - T5z;
+		    T5y = T5t - T5q;
+		    io[WS(ios, 10)] = T5x + T5y;
+		    io[WS(ios, 22)] = T5y - T5x;
+		    T5E = T5B - T5A;
+		    ro[WS(ros, 22)] = T5D - T5E;
+		    ro[WS(ros, 10)] = T5D + T5E;
+	       }
+	  }
+	  {
+	       E T4x, T5j, T5d, T5f, T4O, T5g, T55, T5h;
+	       {
+		    E T4p, T4w, T59, T5c;
+		    T4p = T4l + T4o;
+		    T4w = T4s + T4v;
+		    T4x = T4p + T4w;
+		    T5j = T4p - T4w;
+		    T59 = T57 + T58;
+		    T5c = T5a + T5b;
+		    T5d = T59 - T5c;
+		    T5f = T5c + T59;
+	       }
+	       {
+		    E T4I, T4N, T4Z, T54;
+		    T4I = T4A + T4H;
+		    T4N = T4L + T4M;
+		    T4O = FMA(KP980785280, T4I, KP195090322 * T4N);
+		    T5g = FNMS(KP195090322, T4I, KP980785280 * T4N);
+		    T4Z = T4R + T4Y;
+		    T54 = T52 + T53;
+		    T55 = FNMS(KP195090322, T54, KP980785280 * T4Z);
+		    T5h = FMA(KP195090322, T4Z, KP980785280 * T54);
+	       }
+	       {
+		    E T56, T5i, T5e, T5k;
+		    T56 = T4O + T55;
+		    ro[WS(ros, 30)] = T4x - T56;
+		    ro[WS(ros, 2)] = T4x + T56;
+		    T5i = T5g + T5h;
+		    io[WS(ios, 2)] = T5f + T5i;
+		    io[WS(ios, 30)] = T5i - T5f;
+		    T5e = T55 - T4O;
+		    io[WS(ios, 14)] = T5d + T5e;
+		    io[WS(ios, 18)] = T5e - T5d;
+		    T5k = T5h - T5g;
+		    ro[WS(ros, 18)] = T5j - T5k;
+		    ro[WS(ros, 14)] = T5j + T5k;
+	       }
+	  }
+	  {
+	       E T3p, T41, T4c, T3S, T3w, T4b, T49, T4h, T3P, T42, T3E, T3W, T46, T4g, T3L;
+	       E T3X;
+	       {
+		    E T3s, T3v, T3A, T3D;
+		    T3p = T3n + T3o;
+		    T41 = T3n - T3o;
+		    T4c = T3R - T3Q;
+		    T3S = T3Q + T3R;
+		    T3s = FMA(KP831469612, T3q, KP555570233 * T3r);
+		    T3v = FNMS(KP555570233, T3u, KP831469612 * T3t);
+		    T3w = T3s + T3v;
+		    T4b = T3v - T3s;
+		    {
+			 E T47, T48, T3N, T3O;
+			 T47 = T3F - T3G;
+			 T48 = T3J - T3I;
+			 T49 = FNMS(KP471396736, T48, KP881921264 * T47);
+			 T4h = FMA(KP471396736, T47, KP881921264 * T48);
+			 T3N = FNMS(KP555570233, T3q, KP831469612 * T3r);
+			 T3O = FMA(KP555570233, T3t, KP831469612 * T3u);
+			 T3P = T3N + T3O;
+			 T42 = T3O - T3N;
+		    }
+		    T3A = T3y + T3z;
+		    T3D = T3B + T3C;
+		    T3E = FMA(KP956940335, T3A, KP290284677 * T3D);
+		    T3W = FNMS(KP290284677, T3A, KP956940335 * T3D);
+		    {
+			 E T44, T45, T3H, T3K;
+			 T44 = T3y - T3z;
+			 T45 = T3C - T3B;
+			 T46 = FMA(KP881921264, T44, KP471396736 * T45);
+			 T4g = FNMS(KP471396736, T44, KP881921264 * T45);
+			 T3H = T3F + T3G;
+			 T3K = T3I + T3J;
+			 T3L = FNMS(KP290284677, T3K, KP956940335 * T3H);
+			 T3X = FMA(KP290284677, T3H, KP956940335 * T3K);
+		    }
+	       }
+	       {
+		    E T3x, T3M, T3V, T3Y;
+		    T3x = T3p + T3w;
+		    T3M = T3E + T3L;
+		    ro[WS(ros, 29)] = T3x - T3M;
+		    ro[WS(ros, 3)] = T3x + T3M;
+		    T3V = T3S + T3P;
+		    T3Y = T3W + T3X;
+		    io[WS(ios, 3)] = T3V + T3Y;
+		    io[WS(ios, 29)] = T3Y - T3V;
+	       }
+	       {
+		    E T3T, T3U, T3Z, T40;
+		    T3T = T3P - T3S;
+		    T3U = T3L - T3E;
+		    io[WS(ios, 13)] = T3T + T3U;
+		    io[WS(ios, 19)] = T3U - T3T;
+		    T3Z = T3p - T3w;
+		    T40 = T3X - T3W;
+		    ro[WS(ros, 19)] = T3Z - T40;
+		    ro[WS(ros, 13)] = T3Z + T40;
+	       }
+	       {
+		    E T43, T4a, T4f, T4i;
+		    T43 = T41 + T42;
+		    T4a = T46 + T49;
+		    ro[WS(ros, 27)] = T43 - T4a;
+		    ro[WS(ros, 5)] = T43 + T4a;
+		    T4f = T4c + T4b;
+		    T4i = T4g + T4h;
+		    io[WS(ios, 5)] = T4f + T4i;
+		    io[WS(ios, 27)] = T4i - T4f;
+	       }
+	       {
+		    E T4d, T4e, T4j, T4k;
+		    T4d = T4b - T4c;
+		    T4e = T49 - T46;
+		    io[WS(ios, 11)] = T4d + T4e;
+		    io[WS(ios, 21)] = T4e - T4d;
+		    T4j = T41 - T42;
+		    T4k = T4h - T4g;
+		    ro[WS(ros, 21)] = T4j - T4k;
+		    ro[WS(ros, 11)] = T4j + T4k;
+	       }
+	  }
+	  {
+	       E T1d, T33, T3e, T2U, T1w, T3d, T3b, T3j, T2N, T34, T28, T2Y, T38, T3i, T2J;
+	       E T2Z;
+	       {
+		    E T1m, T1v, T1Y, T27;
+		    T1d = T15 - T1c;
+		    T33 = T15 + T1c;
+		    T3e = T2T + T2Q;
+		    T2U = T2Q - T2T;
+		    T1m = FMA(KP195090322, T1i, KP980785280 * T1l);
+		    T1v = FNMS(KP195090322, T1u, KP980785280 * T1r);
+		    T1w = T1m - T1v;
+		    T3d = T1v + T1m;
+		    {
+			 E T39, T3a, T2L, T2M;
+			 T39 = T2j + T2y;
+			 T3a = T2H + T2C;
+			 T3b = FNMS(KP098017140, T3a, KP995184726 * T39);
+			 T3j = FMA(KP995184726, T3a, KP098017140 * T39);
+			 T2L = FNMS(KP195090322, T1l, KP980785280 * T1i);
+			 T2M = FMA(KP980785280, T1u, KP195090322 * T1r);
+			 T2N = T2L - T2M;
+			 T34 = T2M + T2L;
+		    }
+		    T1Y = T1M - T1X;
+		    T27 = T23 - T26;
+		    T28 = FMA(KP634393284, T1Y, KP773010453 * T27);
+		    T2Y = FNMS(KP634393284, T27, KP773010453 * T1Y);
+		    {
+			 E T36, T37, T2z, T2I;
+			 T36 = T1X + T1M;
+			 T37 = T23 + T26;
+			 T38 = FMA(KP098017140, T36, KP995184726 * T37);
+			 T3i = FNMS(KP098017140, T37, KP995184726 * T36);
+			 T2z = T2j - T2y;
+			 T2I = T2C - T2H;
+			 T2J = FNMS(KP634393284, T2I, KP773010453 * T2z);
+			 T2Z = FMA(KP773010453, T2I, KP634393284 * T2z);
+		    }
+	       }
+	       {
+		    E T1x, T2K, T2X, T30;
+		    T1x = T1d + T1w;
+		    T2K = T28 + T2J;
+		    ro[WS(ros, 25)] = T1x - T2K;
+		    ro[WS(ros, 7)] = T1x + T2K;
+		    T2X = T2U + T2N;
+		    T30 = T2Y + T2Z;
+		    io[WS(ios, 7)] = T2X + T30;
+		    io[WS(ios, 25)] = T30 - T2X;
+	       }
+	       {
+		    E T2V, T2W, T31, T32;
+		    T2V = T2N - T2U;
+		    T2W = T2J - T28;
+		    io[WS(ios, 9)] = T2V + T2W;
+		    io[WS(ios, 23)] = T2W - T2V;
+		    T31 = T1d - T1w;
+		    T32 = T2Z - T2Y;
+		    ro[WS(ros, 23)] = T31 - T32;
+		    ro[WS(ros, 9)] = T31 + T32;
+	       }
+	       {
+		    E T35, T3c, T3h, T3k;
+		    T35 = T33 + T34;
+		    T3c = T38 + T3b;
+		    ro[WS(ros, 31)] = T35 - T3c;
+		    ro[WS(ros, 1)] = T35 + T3c;
+		    T3h = T3e + T3d;
+		    T3k = T3i + T3j;
+		    io[WS(ios, 1)] = T3h + T3k;
+		    io[WS(ios, 31)] = T3k - T3h;
+	       }
+	       {
+		    E T3f, T3g, T3l, T3m;
+		    T3f = T3d - T3e;
+		    T3g = T3b - T38;
+		    io[WS(ios, 15)] = T3f + T3g;
+		    io[WS(ios, 17)] = T3g - T3f;
+		    T3l = T33 - T34;
+		    T3m = T3j - T3i;
+		    ro[WS(ros, 17)] = T3l - T3m;
+		    ro[WS(ros, 15)] = T3l + T3m;
+	       }
+	  }
+     }
+}
+
+static void mr2hc_64(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  mr2hc_64_0(I, ro, io, is, ros, ios);
+	  I += ivs;
+	  ro += ovs;
+	  io += ovs;
+     }
+}
+
+static const kr2hc_desc desc = { 64, "mr2hc_64", {342, 72, 52, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_mr2hc_64) (planner *p) {
+     X(kr2hc_register) (p, mr2hc_64, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_10.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_10.c
new file mode 100644
index 0000000..d163e75
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_10.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:12 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 10 -name r2hcII_10 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 32 FP additions, 12 FP multiplications,
+ * (or, 26 additions, 6 multiplications, 6 fused multiply/add),
+ * 21 stack variables, and 20 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_10.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_10.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_10.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_10(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, To, T8, Tq, T9, Tp, Te, Ts, Th, Tn;
+	  T1 = I[0];
+	  To = I[WS(is, 5)];
+	  {
+	       E T2, T3, T4, T5, T6, T7;
+	       T2 = I[WS(is, 4)];
+	       T3 = I[WS(is, 6)];
+	       T4 = T2 - T3;
+	       T5 = I[WS(is, 8)];
+	       T6 = I[WS(is, 2)];
+	       T7 = T5 - T6;
+	       T8 = T4 + T7;
+	       Tq = T5 + T6;
+	       T9 = KP559016994 * (T4 - T7);
+	       Tp = T2 + T3;
+	  }
+	  {
+	       E Tc, Td, Tm, Tf, Tg, Tl;
+	       Tc = I[WS(is, 1)];
+	       Td = I[WS(is, 9)];
+	       Tm = Tc + Td;
+	       Tf = I[WS(is, 3)];
+	       Tg = I[WS(is, 7)];
+	       Tl = Tf + Tg;
+	       Te = Tc - Td;
+	       Ts = KP559016994 * (Tm + Tl);
+	       Th = Tf - Tg;
+	       Tn = Tl - Tm;
+	  }
+	  ro[WS(ros, 2)] = T1 + T8;
+	  io[WS(ios, 2)] = Tn - To;
+	  {
+	       E Ti, Tk, Tb, Tj, Ta;
+	       Ti = FMA(KP951056516, Te, KP587785252 * Th);
+	       Tk = FNMS(KP587785252, Te, KP951056516 * Th);
+	       Ta = FNMS(KP250000000, T8, T1);
+	       Tb = T9 + Ta;
+	       Tj = Ta - T9;
+	       ro[WS(ros, 4)] = Tb - Ti;
+	       ro[WS(ros, 3)] = Tj + Tk;
+	       ro[0] = Tb + Ti;
+	       ro[WS(ros, 1)] = Tj - Tk;
+	  }
+	  {
+	       E Tr, Tw, Tu, Tv, Tt;
+	       Tr = FMA(KP951056516, Tp, KP587785252 * Tq);
+	       Tw = FNMS(KP587785252, Tp, KP951056516 * Tq);
+	       Tt = FMA(KP250000000, Tn, To);
+	       Tu = Ts + Tt;
+	       Tv = Tt - Ts;
+	       io[0] = -(Tr + Tu);
+	       io[WS(ios, 3)] = Tw + Tv;
+	       io[WS(ios, 4)] = Tr - Tu;
+	       io[WS(ios, 1)] = Tv - Tw;
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 10, "r2hcII_10", {26, 6, 6, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_10) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_10, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_12.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_12.c
new file mode 100644
index 0000000..9e01c7b
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_12.c
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:14 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 12 -name r2hcII_12 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 43 FP additions, 12 FP multiplications,
+ * (or, 39 additions, 8 multiplications, 4 fused multiply/add),
+ * 28 stack variables, and 24 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_12.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_12.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_12.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_12(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP353553390, +0.353553390593273762200422181052424519642417969);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     DK(KP612372435, +0.612372435695794524549321018676472847991486870);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E Tx, Tg, T4, Tz, Ty, Tj, TA, T9, Tm, Tl, Te, Tp, To, Tf, TE;
+	  E TF;
+	  {
+	       E T1, T3, T2, Th, Ti;
+	       T1 = I[0];
+	       T3 = I[WS(is, 4)];
+	       T2 = I[WS(is, 8)];
+	       Tx = KP866025403 * (T2 + T3);
+	       Tg = FMA(KP500000000, T3 - T2, T1);
+	       T4 = T1 + T2 - T3;
+	       Tz = I[WS(is, 6)];
+	       Th = I[WS(is, 10)];
+	       Ti = I[WS(is, 2)];
+	       Ty = Th + Ti;
+	       Tj = KP866025403 * (Th - Ti);
+	       TA = FMA(KP500000000, Ty, Tz);
+	  }
+	  {
+	       E T5, T6, T7, T8;
+	       T5 = I[WS(is, 3)];
+	       T6 = I[WS(is, 11)];
+	       T7 = I[WS(is, 7)];
+	       T8 = T6 - T7;
+	       T9 = T5 + T8;
+	       Tm = KP612372435 * (T6 + T7);
+	       Tl = FNMS(KP353553390, T8, KP707106781 * T5);
+	  }
+	  {
+	       E Td, Ta, Tb, Tc;
+	       Td = I[WS(is, 9)];
+	       Ta = I[WS(is, 5)];
+	       Tb = I[WS(is, 1)];
+	       Tc = Ta - Tb;
+	       Te = Tc - Td;
+	       Tp = FMA(KP353553390, Tc, KP707106781 * Td);
+	       To = KP612372435 * (Ta + Tb);
+	  }
+	  Tf = KP707106781 * (T9 + Te);
+	  ro[WS(ros, 1)] = T4 - Tf;
+	  ro[WS(ros, 4)] = T4 + Tf;
+	  TE = KP707106781 * (Te - T9);
+	  TF = Tz - Ty;
+	  io[WS(ios, 4)] = TE - TF;
+	  io[WS(ios, 1)] = TE + TF;
+	  {
+	       E Tk, TB, Tr, Tw, Tn, Tq;
+	       Tk = Tg - Tj;
+	       TB = Tx - TA;
+	       Tn = Tl - Tm;
+	       Tq = To - Tp;
+	       Tr = Tn + Tq;
+	       Tw = Tn - Tq;
+	       ro[WS(ros, 5)] = Tk - Tr;
+	       io[WS(ios, 2)] = Tw + TB;
+	       ro[0] = Tk + Tr;
+	       io[WS(ios, 3)] = Tw - TB;
+	  }
+	  {
+	       E Ts, TD, Tv, TC, Tt, Tu;
+	       Ts = Tg + Tj;
+	       TD = Tx + TA;
+	       Tt = To + Tp;
+	       Tu = Tm + Tl;
+	       Tv = Tt - Tu;
+	       TC = Tu + Tt;
+	       ro[WS(ros, 3)] = Ts - Tv;
+	       io[WS(ios, 5)] = TD - TC;
+	       ro[WS(ros, 2)] = Ts + Tv;
+	       io[0] = -(TC + TD);
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 12, "r2hcII_12", {39, 8, 4, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_12) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_12, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_15.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_15.c
new file mode 100644
index 0000000..72e014a
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_15.c
@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:16 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 15 -name r2hcII_15 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 72 FP additions, 33 FP multiplications,
+ * (or, 54 additions, 15 multiplications, 18 fused multiply/add),
+ * 37 stack variables, and 30 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_15.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_15.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_15.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_15(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP809016994, +0.809016994374947424102293417182819058860154590);
+     DK(KP309016994, +0.309016994374947424102293417182819058860154590);
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T2, Tx, TR, TE, T7, TD, Th, Tm, Tr, TQ, TA, TB, Tf, Te;
+	  E Tu, TS, Td, TH, TO;
+	  T1 = I[WS(is, 10)];
+	  {
+	       E T3, Tv, T6, Tw, T4, T5;
+	       T2 = I[WS(is, 4)];
+	       T3 = I[WS(is, 1)];
+	       Tv = T2 + T3;
+	       T4 = I[WS(is, 7)];
+	       T5 = I[WS(is, 13)];
+	       T6 = T4 + T5;
+	       Tw = T4 - T5;
+	       Tx = FMA(KP951056516, Tv, KP587785252 * Tw);
+	       TR = FNMS(KP587785252, Tv, KP951056516 * Tw);
+	       TE = KP559016994 * (T3 - T6);
+	       T7 = T3 + T6;
+	       TD = KP250000000 * T7;
+	  }
+	  {
+	       E Ti, Tl, Tj, Tk, Tp, Tq;
+	       Th = I[0];
+	       Ti = I[WS(is, 9)];
+	       Tl = I[WS(is, 12)];
+	       Tj = I[WS(is, 3)];
+	       Tk = I[WS(is, 6)];
+	       Tp = Tk + Ti;
+	       Tq = Tl + Tj;
+	       Tm = Ti + Tj - (Tk + Tl);
+	       Tr = FMA(KP951056516, Tp, KP587785252 * Tq);
+	       TQ = FNMS(KP951056516, Tq, KP587785252 * Tp);
+	       TA = FMA(KP250000000, Tm, Th);
+	       TB = KP559016994 * (Tl + Ti - (Tk + Tj));
+	  }
+	  {
+	       E T9, Tt, Tc, Ts, Ta, Tb, TG;
+	       Tf = I[WS(is, 5)];
+	       T9 = I[WS(is, 14)];
+	       Te = I[WS(is, 11)];
+	       Tt = T9 + Te;
+	       Ta = I[WS(is, 2)];
+	       Tb = I[WS(is, 8)];
+	       Tc = Ta + Tb;
+	       Ts = Ta - Tb;
+	       Tu = FNMS(KP951056516, Tt, KP587785252 * Ts);
+	       TS = FMA(KP951056516, Ts, KP587785252 * Tt);
+	       Td = T9 + Tc;
+	       TG = KP559016994 * (T9 - Tc);
+	       TH = FNMS(KP309016994, Te, TG) + FNMA(KP250000000, Td, Tf);
+	       TO = FMS(KP809016994, Te, Tf) + FNMA(KP250000000, Td, TG);
+	  }
+	  {
+	       E Tn, T8, Tg, To;
+	       Tn = Th - Tm;
+	       T8 = T1 + T2 - T7;
+	       Tg = Td - Te - Tf;
+	       To = T8 + Tg;
+	       io[WS(ios, 2)] = KP866025403 * (T8 - Tg);
+	       ro[WS(ros, 2)] = FNMS(KP500000000, To, Tn);
+	       ro[WS(ros, 7)] = Tn + To;
+	  }
+	  {
+	       E TM, TX, TT, TV, TP, TU, TN, TW;
+	       TM = TB + TA;
+	       TX = KP866025403 * (TR + TS);
+	       TT = TR - TS;
+	       TV = FMS(KP500000000, TT, TQ);
+	       TN = T1 + TE + FNMS(KP809016994, T2, TD);
+	       TP = TN + TO;
+	       TU = KP866025403 * (TO - TN);
+	       ro[WS(ros, 1)] = TM + TP;
+	       io[WS(ios, 1)] = TQ + TT;
+	       io[WS(ios, 6)] = TU - TV;
+	       io[WS(ios, 3)] = TU + TV;
+	       TW = FNMS(KP500000000, TP, TM);
+	       ro[WS(ros, 3)] = TW - TX;
+	       ro[WS(ros, 6)] = TW + TX;
+	  }
+	  {
+	       E Tz, TC, Ty, TK, TI, TL, TF, TJ;
+	       Tz = KP866025403 * (Tx + Tu);
+	       TC = TA - TB;
+	       Ty = Tu - Tx;
+	       TK = FMS(KP500000000, Ty, Tr);
+	       TF = FMA(KP309016994, T2, T1) + TD - TE;
+	       TI = TF + TH;
+	       TL = KP866025403 * (TH - TF);
+	       io[WS(ios, 4)] = Tr + Ty;
+	       ro[WS(ros, 4)] = TC + TI;
+	       io[WS(ios, 5)] = TK - TL;
+	       io[0] = TK + TL;
+	       TJ = FNMS(KP500000000, TI, TC);
+	       ro[0] = Tz + TJ;
+	       ro[WS(ros, 5)] = TJ - Tz;
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 15, "r2hcII_15", {54, 15, 18, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_15) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_15, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_16.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_16.c
new file mode 100644
index 0000000..4535bde
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_16.c
@@ -0,0 +1,171 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:21 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 16 -name r2hcII_16 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 66 FP additions, 30 FP multiplications,
+ * (or, 54 additions, 18 multiplications, 12 fused multiply/add),
+ * 32 stack variables, and 32 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_16(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T5, T11, TB, TV, Tr, TK, Tu, TJ, Ti, TH, Tl, TG, Tc, T10, TE;
+	  E TS;
+	  {
+	       E T1, TU, T4, TT, T2, T3;
+	       T1 = I[0];
+	       TU = I[WS(is, 8)];
+	       T2 = I[WS(is, 4)];
+	       T3 = I[WS(is, 12)];
+	       T4 = KP707106781 * (T2 - T3);
+	       TT = KP707106781 * (T2 + T3);
+	       T5 = T1 + T4;
+	       T11 = TU - TT;
+	       TB = T1 - T4;
+	       TV = TT + TU;
+	  }
+	  {
+	       E Tq, Tt, Tp, Ts, Tn, To;
+	       Tq = I[WS(is, 15)];
+	       Tt = I[WS(is, 7)];
+	       Tn = I[WS(is, 3)];
+	       To = I[WS(is, 11)];
+	       Tp = KP707106781 * (Tn - To);
+	       Ts = KP707106781 * (Tn + To);
+	       Tr = Tp - Tq;
+	       TK = Tt - Ts;
+	       Tu = Ts + Tt;
+	       TJ = Tp + Tq;
+	  }
+	  {
+	       E Te, Tk, Th, Tj, Tf, Tg;
+	       Te = I[WS(is, 1)];
+	       Tk = I[WS(is, 9)];
+	       Tf = I[WS(is, 5)];
+	       Tg = I[WS(is, 13)];
+	       Th = KP707106781 * (Tf - Tg);
+	       Tj = KP707106781 * (Tf + Tg);
+	       Ti = Te + Th;
+	       TH = Tk - Tj;
+	       Tl = Tj + Tk;
+	       TG = Te - Th;
+	  }
+	  {
+	       E T8, TC, Tb, TD;
+	       {
+		    E T6, T7, T9, Ta;
+		    T6 = I[WS(is, 2)];
+		    T7 = I[WS(is, 10)];
+		    T8 = FNMS(KP382683432, T7, KP923879532 * T6);
+		    TC = FMA(KP382683432, T6, KP923879532 * T7);
+		    T9 = I[WS(is, 6)];
+		    Ta = I[WS(is, 14)];
+		    Tb = FNMS(KP923879532, Ta, KP382683432 * T9);
+		    TD = FMA(KP923879532, T9, KP382683432 * Ta);
+	       }
+	       Tc = T8 + Tb;
+	       T10 = Tb - T8;
+	       TE = TC - TD;
+	       TS = TC + TD;
+	  }
+	  {
+	       E Td, TW, Tw, TR, Tm, Tv;
+	       Td = T5 - Tc;
+	       TW = TS + TV;
+	       Tm = FMA(KP195090322, Ti, KP980785280 * Tl);
+	       Tv = FNMS(KP980785280, Tu, KP195090322 * Tr);
+	       Tw = Tm + Tv;
+	       TR = Tv - Tm;
+	       ro[WS(ros, 4)] = Td - Tw;
+	       io[WS(ios, 7)] = TR + TW;
+	       ro[WS(ros, 3)] = Td + Tw;
+	       io[0] = TR - TW;
+	  }
+	  {
+	       E Tx, TY, TA, TX, Ty, Tz;
+	       Tx = T5 + Tc;
+	       TY = TV - TS;
+	       Ty = FNMS(KP195090322, Tl, KP980785280 * Ti);
+	       Tz = FMA(KP980785280, Tr, KP195090322 * Tu);
+	       TA = Ty + Tz;
+	       TX = Tz - Ty;
+	       ro[WS(ros, 7)] = Tx - TA;
+	       io[WS(ios, 3)] = TX + TY;
+	       ro[0] = Tx + TA;
+	       io[WS(ios, 4)] = TX - TY;
+	  }
+	  {
+	       E TF, T12, TM, TZ, TI, TL;
+	       TF = TB + TE;
+	       T12 = T10 - T11;
+	       TI = FMA(KP831469612, TG, KP555570233 * TH);
+	       TL = FMA(KP831469612, TJ, KP555570233 * TK);
+	       TM = TI - TL;
+	       TZ = TI + TL;
+	       ro[WS(ros, 6)] = TF - TM;
+	       io[WS(ios, 2)] = T12 - TZ;
+	       ro[WS(ros, 1)] = TF + TM;
+	       io[WS(ios, 5)] = -(TZ + T12);
+	  }
+	  {
+	       E TN, T14, TQ, T13, TO, TP;
+	       TN = TB - TE;
+	       T14 = T10 + T11;
+	       TO = FNMS(KP555570233, TJ, KP831469612 * TK);
+	       TP = FNMS(KP555570233, TG, KP831469612 * TH);
+	       TQ = TO - TP;
+	       T13 = TP + TO;
+	       ro[WS(ros, 5)] = TN - TQ;
+	       io[WS(ios, 1)] = T13 + T14;
+	       ro[WS(ros, 2)] = TN + TQ;
+	       io[WS(ios, 6)] = T13 - T14;
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 16, "r2hcII_16", {54, 18, 12, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_16) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_16, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_2.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_2.c
new file mode 100644
index 0000000..00ca93b
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_2.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:02 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 2 -name r2hcII_2 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 0 FP additions, 0 FP multiplications,
+ * (or, 0 additions, 0 multiplications, 0 fused multiply/add),
+ * 3 stack variables, and 4 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_2.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_2.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_2.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_2(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T2;
+	  T1 = I[0];
+	  ro[0] = T1;
+	  T2 = I[WS(is, 1)];
+	  io[0] = -T2;
+     }
+}
+
+static const kr2hc_desc desc = { 2, "r2hcII_2", {0, 0, 0, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_2) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_2, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_3.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_3.c
new file mode 100644
index 0000000..afdc6d1
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_3.c
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:02 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 3 -name r2hcII_3 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 4 FP additions, 2 FP multiplications,
+ * (or, 3 additions, 1 multiplications, 1 fused multiply/add),
+ * 7 stack variables, and 6 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_3.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_3.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_3.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_3(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T2, T3, T4;
+	  T1 = I[0];
+	  T2 = I[WS(is, 1)];
+	  T3 = I[WS(is, 2)];
+	  T4 = T2 - T3;
+	  ro[WS(ros, 1)] = T1 - T4;
+	  io[0] = -(KP866025403 * (T2 + T3));
+	  ro[0] = FMA(KP500000000, T4, T1);
+     }
+}
+
+static const kr2hc_desc desc = { 3, "r2hcII_3", {3, 1, 1, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_3) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_3, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_32.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_32.c
new file mode 100644
index 0000000..728ae3d
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_32.c
@@ -0,0 +1,358 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:28 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 32 -name r2hcII_32 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 174 FP additions, 82 FP multiplications,
+ * (or, 138 additions, 46 multiplications, 36 fused multiply/add),
+ * 62 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_32(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP471396736, +0.471396736825997648556387625905254377657460319);
+     DK(KP881921264, +0.881921264348355029712756863660388349508442621);
+     DK(KP634393284, +0.634393284163645498215171613225493370675687095);
+     DK(KP773010453, +0.773010453362736960810906609758469800971041293);
+     DK(KP290284677, +0.290284677254462367636192375817395274691476278);
+     DK(KP956940335, +0.956940335732208864935797886980269969482849206);
+     DK(KP995184726, +0.995184726672196886244836953109479921575474869);
+     DK(KP098017140, +0.098017140329560601994195563888641845861136673);
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T5, T2D, T1z, T2q, Tc, T2C, T1C, T2n, Tm, T1k, T1J, T26, Tv, T1l, T1G;
+	  E T27, T15, T1r, T1Y, T2e, T1c, T1s, T1V, T2d, TK, T1o, T1R, T2b, TR, T1p;
+	  E T1O, T2a;
+	  {
+	       E T1, T2p, T4, T2o, T2, T3;
+	       T1 = I[0];
+	       T2p = I[WS(is, 16)];
+	       T2 = I[WS(is, 8)];
+	       T3 = I[WS(is, 24)];
+	       T4 = KP707106781 * (T2 - T3);
+	       T2o = KP707106781 * (T2 + T3);
+	       T5 = T1 + T4;
+	       T2D = T2p - T2o;
+	       T1z = T1 - T4;
+	       T2q = T2o + T2p;
+	  }
+	  {
+	       E T8, T1A, Tb, T1B;
+	       {
+		    E T6, T7, T9, Ta;
+		    T6 = I[WS(is, 4)];
+		    T7 = I[WS(is, 20)];
+		    T8 = FNMS(KP382683432, T7, KP923879532 * T6);
+		    T1A = FMA(KP382683432, T6, KP923879532 * T7);
+		    T9 = I[WS(is, 12)];
+		    Ta = I[WS(is, 28)];
+		    Tb = FNMS(KP923879532, Ta, KP382683432 * T9);
+		    T1B = FMA(KP923879532, T9, KP382683432 * Ta);
+	       }
+	       Tc = T8 + Tb;
+	       T2C = Tb - T8;
+	       T1C = T1A - T1B;
+	       T2n = T1A + T1B;
+	  }
+	  {
+	       E Te, Tk, Th, Tj, Tf, Tg;
+	       Te = I[WS(is, 2)];
+	       Tk = I[WS(is, 18)];
+	       Tf = I[WS(is, 10)];
+	       Tg = I[WS(is, 26)];
+	       Th = KP707106781 * (Tf - Tg);
+	       Tj = KP707106781 * (Tf + Tg);
+	       {
+		    E Ti, Tl, T1H, T1I;
+		    Ti = Te + Th;
+		    Tl = Tj + Tk;
+		    Tm = FNMS(KP195090322, Tl, KP980785280 * Ti);
+		    T1k = FMA(KP195090322, Ti, KP980785280 * Tl);
+		    T1H = Tk - Tj;
+		    T1I = Te - Th;
+		    T1J = FNMS(KP555570233, T1I, KP831469612 * T1H);
+		    T26 = FMA(KP831469612, T1I, KP555570233 * T1H);
+	       }
+	  }
+	  {
+	       E Tq, Tt, Tp, Ts, Tn, To;
+	       Tq = I[WS(is, 30)];
+	       Tt = I[WS(is, 14)];
+	       Tn = I[WS(is, 6)];
+	       To = I[WS(is, 22)];
+	       Tp = KP707106781 * (Tn - To);
+	       Ts = KP707106781 * (Tn + To);
+	       {
+		    E Tr, Tu, T1E, T1F;
+		    Tr = Tp - Tq;
+		    Tu = Ts + Tt;
+		    Tv = FMA(KP980785280, Tr, KP195090322 * Tu);
+		    T1l = FNMS(KP980785280, Tu, KP195090322 * Tr);
+		    T1E = Tt - Ts;
+		    T1F = Tp + Tq;
+		    T1G = FNMS(KP555570233, T1F, KP831469612 * T1E);
+		    T27 = FMA(KP831469612, T1F, KP555570233 * T1E);
+	       }
+	  }
+	  {
+	       E TW, T1a, TV, T19, T10, T16, T13, T17, TT, TU;
+	       TW = I[WS(is, 31)];
+	       T1a = I[WS(is, 15)];
+	       TT = I[WS(is, 7)];
+	       TU = I[WS(is, 23)];
+	       TV = KP707106781 * (TT - TU);
+	       T19 = KP707106781 * (TT + TU);
+	       {
+		    E TY, TZ, T11, T12;
+		    TY = I[WS(is, 3)];
+		    TZ = I[WS(is, 19)];
+		    T10 = FNMS(KP382683432, TZ, KP923879532 * TY);
+		    T16 = FMA(KP382683432, TY, KP923879532 * TZ);
+		    T11 = I[WS(is, 11)];
+		    T12 = I[WS(is, 27)];
+		    T13 = FNMS(KP923879532, T12, KP382683432 * T11);
+		    T17 = FMA(KP923879532, T11, KP382683432 * T12);
+	       }
+	       {
+		    E TX, T14, T1W, T1X;
+		    TX = TV - TW;
+		    T14 = T10 + T13;
+		    T15 = TX + T14;
+		    T1r = TX - T14;
+		    T1W = T13 - T10;
+		    T1X = T1a - T19;
+		    T1Y = T1W - T1X;
+		    T2e = T1W + T1X;
+	       }
+	       {
+		    E T18, T1b, T1T, T1U;
+		    T18 = T16 + T17;
+		    T1b = T19 + T1a;
+		    T1c = T18 + T1b;
+		    T1s = T1b - T18;
+		    T1T = TV + TW;
+		    T1U = T16 - T17;
+		    T1V = T1T + T1U;
+		    T2d = T1U - T1T;
+	       }
+	  }
+	  {
+	       E Ty, TP, TB, TO, TF, TL, TI, TM, Tz, TA;
+	       Ty = I[WS(is, 1)];
+	       TP = I[WS(is, 17)];
+	       Tz = I[WS(is, 9)];
+	       TA = I[WS(is, 25)];
+	       TB = KP707106781 * (Tz - TA);
+	       TO = KP707106781 * (Tz + TA);
+	       {
+		    E TD, TE, TG, TH;
+		    TD = I[WS(is, 5)];
+		    TE = I[WS(is, 21)];
+		    TF = FNMS(KP382683432, TE, KP923879532 * TD);
+		    TL = FMA(KP382683432, TD, KP923879532 * TE);
+		    TG = I[WS(is, 13)];
+		    TH = I[WS(is, 29)];
+		    TI = FNMS(KP923879532, TH, KP382683432 * TG);
+		    TM = FMA(KP923879532, TG, KP382683432 * TH);
+	       }
+	       {
+		    E TC, TJ, T1P, T1Q;
+		    TC = Ty + TB;
+		    TJ = TF + TI;
+		    TK = TC + TJ;
+		    T1o = TC - TJ;
+		    T1P = TI - TF;
+		    T1Q = TP - TO;
+		    T1R = T1P - T1Q;
+		    T2b = T1P + T1Q;
+	       }
+	       {
+		    E TN, TQ, T1M, T1N;
+		    TN = TL + TM;
+		    TQ = TO + TP;
+		    TR = TN + TQ;
+		    T1p = TQ - TN;
+		    T1M = Ty - TB;
+		    T1N = TL - TM;
+		    T1O = T1M - T1N;
+		    T2a = T1M + T1N;
+	       }
+	  }
+	  {
+	       E Tx, T1f, T2s, T2u, T1e, T2l, T1i, T2t;
+	       {
+		    E Td, Tw, T2m, T2r;
+		    Td = T5 + Tc;
+		    Tw = Tm + Tv;
+		    Tx = Td - Tw;
+		    T1f = Td + Tw;
+		    T2m = T1l - T1k;
+		    T2r = T2n + T2q;
+		    T2s = T2m - T2r;
+		    T2u = T2m + T2r;
+	       }
+	       {
+		    E TS, T1d, T1g, T1h;
+		    TS = FMA(KP098017140, TK, KP995184726 * TR);
+		    T1d = FNMS(KP995184726, T1c, KP098017140 * T15);
+		    T1e = TS + T1d;
+		    T2l = T1d - TS;
+		    T1g = FNMS(KP098017140, TR, KP995184726 * TK);
+		    T1h = FMA(KP995184726, T15, KP098017140 * T1c);
+		    T1i = T1g + T1h;
+		    T2t = T1h - T1g;
+	       }
+	       ro[WS(ros, 8)] = Tx - T1e;
+	       io[WS(ios, 8)] = T2t - T2u;
+	       ro[WS(ros, 7)] = Tx + T1e;
+	       io[WS(ios, 7)] = T2t + T2u;
+	       ro[WS(ros, 15)] = T1f - T1i;
+	       io[WS(ios, 15)] = T2l - T2s;
+	       ro[0] = T1f + T1i;
+	       io[0] = T2l + T2s;
+	  }
+	  {
+	       E T29, T2h, T2M, T2O, T2g, T2J, T2k, T2N;
+	       {
+		    E T25, T28, T2K, T2L;
+		    T25 = T1z + T1C;
+		    T28 = T26 - T27;
+		    T29 = T25 + T28;
+		    T2h = T25 - T28;
+		    T2K = T1J + T1G;
+		    T2L = T2C + T2D;
+		    T2M = T2K - T2L;
+		    T2O = T2K + T2L;
+	       }
+	       {
+		    E T2c, T2f, T2i, T2j;
+		    T2c = FMA(KP956940335, T2a, KP290284677 * T2b);
+		    T2f = FNMS(KP290284677, T2e, KP956940335 * T2d);
+		    T2g = T2c + T2f;
+		    T2J = T2f - T2c;
+		    T2i = FMA(KP290284677, T2d, KP956940335 * T2e);
+		    T2j = FNMS(KP290284677, T2a, KP956940335 * T2b);
+		    T2k = T2i - T2j;
+		    T2N = T2j + T2i;
+	       }
+	       ro[WS(ros, 14)] = T29 - T2g;
+	       io[WS(ios, 14)] = T2N - T2O;
+	       ro[WS(ros, 1)] = T29 + T2g;
+	       io[WS(ios, 1)] = T2N + T2O;
+	       ro[WS(ros, 9)] = T2h - T2k;
+	       io[WS(ios, 9)] = T2J - T2M;
+	       ro[WS(ros, 6)] = T2h + T2k;
+	       io[WS(ios, 6)] = T2J + T2M;
+	  }
+	  {
+	       E T1n, T1v, T2y, T2A, T1u, T2v, T1y, T2z;
+	       {
+		    E T1j, T1m, T2w, T2x;
+		    T1j = T5 - Tc;
+		    T1m = T1k + T1l;
+		    T1n = T1j + T1m;
+		    T1v = T1j - T1m;
+		    T2w = Tv - Tm;
+		    T2x = T2q - T2n;
+		    T2y = T2w - T2x;
+		    T2A = T2w + T2x;
+	       }
+	       {
+		    E T1q, T1t, T1w, T1x;
+		    T1q = FMA(KP773010453, T1o, KP634393284 * T1p);
+		    T1t = FNMS(KP634393284, T1s, KP773010453 * T1r);
+		    T1u = T1q + T1t;
+		    T2v = T1t - T1q;
+		    T1w = FMA(KP634393284, T1r, KP773010453 * T1s);
+		    T1x = FNMS(KP634393284, T1o, KP773010453 * T1p);
+		    T1y = T1w - T1x;
+		    T2z = T1x + T1w;
+	       }
+	       ro[WS(ros, 12)] = T1n - T1u;
+	       io[WS(ios, 12)] = T2z - T2A;
+	       ro[WS(ros, 3)] = T1n + T1u;
+	       io[WS(ios, 3)] = T2z + T2A;
+	       ro[WS(ros, 11)] = T1v - T1y;
+	       io[WS(ios, 11)] = T2v - T2y;
+	       ro[WS(ros, 4)] = T1v + T1y;
+	       io[WS(ios, 4)] = T2v + T2y;
+	  }
+	  {
+	       E T1L, T21, T2G, T2I, T20, T2H, T24, T2B;
+	       {
+		    E T1D, T1K, T2E, T2F;
+		    T1D = T1z - T1C;
+		    T1K = T1G - T1J;
+		    T1L = T1D + T1K;
+		    T21 = T1D - T1K;
+		    T2E = T2C - T2D;
+		    T2F = T26 + T27;
+		    T2G = T2E - T2F;
+		    T2I = T2F + T2E;
+	       }
+	       {
+		    E T1S, T1Z, T22, T23;
+		    T1S = FMA(KP881921264, T1O, KP471396736 * T1R);
+		    T1Z = FMA(KP881921264, T1V, KP471396736 * T1Y);
+		    T20 = T1S - T1Z;
+		    T2H = T1S + T1Z;
+		    T22 = FNMS(KP471396736, T1V, KP881921264 * T1Y);
+		    T23 = FNMS(KP471396736, T1O, KP881921264 * T1R);
+		    T24 = T22 - T23;
+		    T2B = T23 + T22;
+	       }
+	       ro[WS(ros, 13)] = T1L - T20;
+	       io[WS(ios, 13)] = T2B - T2G;
+	       ro[WS(ros, 2)] = T1L + T20;
+	       io[WS(ios, 2)] = T2B + T2G;
+	       ro[WS(ros, 10)] = T21 - T24;
+	       io[WS(ios, 10)] = T2I - T2H;
+	       ro[WS(ros, 5)] = T21 + T24;
+	       io[WS(ios, 5)] = -(T2H + T2I);
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 32, "r2hcII_32", {138, 46, 36, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_32) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_32, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_4.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_4.c
new file mode 100644
index 0000000..94aaf84
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_4.c
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:02 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 4 -name r2hcII_4 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 6 FP additions, 2 FP multiplications,
+ * (or, 6 additions, 2 multiplications, 0 fused multiply/add),
+ * 8 stack variables, and 8 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_4(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T6, T4, T5, T2, T3;
+	  T1 = I[0];
+	  T6 = I[WS(is, 2)];
+	  T2 = I[WS(is, 1)];
+	  T3 = I[WS(is, 3)];
+	  T4 = KP707106781 * (T2 - T3);
+	  T5 = KP707106781 * (T2 + T3);
+	  ro[WS(ros, 1)] = T1 - T4;
+	  io[WS(ios, 1)] = T6 - T5;
+	  ro[0] = T1 + T4;
+	  io[0] = -(T5 + T6);
+     }
+}
+
+static const kr2hc_desc desc = { 4, "r2hcII_4", {6, 2, 0, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_4) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_4, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_5.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_5.c
new file mode 100644
index 0000000..bb40e64
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_5.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:03 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 5 -name r2hcII_5 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 12 FP additions, 6 FP multiplications,
+ * (or, 9 additions, 3 multiplications, 3 fused multiply/add),
+ * 17 stack variables, and 10 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_5.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_5.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_5.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_5(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T8, T3, T6, T9, Tc, Tb, T7, Ta;
+	  T8 = I[0];
+	  {
+	       E T1, T2, T4, T5;
+	       T1 = I[WS(is, 2)];
+	       T2 = I[WS(is, 3)];
+	       T3 = T1 - T2;
+	       T4 = I[WS(is, 4)];
+	       T5 = I[WS(is, 1)];
+	       T6 = T4 - T5;
+	       T9 = T3 + T6;
+	       Tc = T4 + T5;
+	       Tb = T1 + T2;
+	  }
+	  ro[WS(ros, 2)] = T8 + T9;
+	  io[WS(ios, 1)] = FNMS(KP951056516, Tc, KP587785252 * Tb);
+	  io[0] = -(FMA(KP951056516, Tb, KP587785252 * Tc));
+	  T7 = KP559016994 * (T3 - T6);
+	  Ta = FNMS(KP250000000, T9, T8);
+	  ro[0] = T7 + Ta;
+	  ro[WS(ros, 1)] = Ta - T7;
+     }
+}
+
+static const kr2hc_desc desc = { 5, "r2hcII_5", {9, 3, 3, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_5) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_5, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_6.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_6.c
new file mode 100644
index 0000000..04e76c9
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_6.c
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:03 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 6 -name r2hcII_6 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 13 FP additions, 4 FP multiplications,
+ * (or, 11 additions, 2 multiplications, 2 fused multiply/add),
+ * 14 stack variables, and 12 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_6.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_6.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_6.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_6(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E Ta, T7, T9, T1, T3, T2, T8, T4, T5, T6, Tb;
+	  Ta = I[WS(is, 3)];
+	  T5 = I[WS(is, 5)];
+	  T6 = I[WS(is, 1)];
+	  T7 = KP866025403 * (T5 - T6);
+	  T9 = T5 + T6;
+	  T1 = I[0];
+	  T3 = I[WS(is, 2)];
+	  T2 = I[WS(is, 4)];
+	  T8 = KP866025403 * (T2 + T3);
+	  T4 = FMA(KP500000000, T3 - T2, T1);
+	  ro[0] = T4 - T7;
+	  ro[WS(ros, 2)] = T4 + T7;
+	  io[WS(ios, 1)] = Ta - T9;
+	  ro[WS(ros, 1)] = T1 + T2 - T3;
+	  Tb = FMA(KP500000000, T9, Ta);
+	  io[0] = -(T8 + Tb);
+	  io[WS(ios, 2)] = T8 - Tb;
+     }
+}
+
+static const kr2hc_desc desc = { 6, "r2hcII_6", {11, 2, 2, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_6) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_6, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_7.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_7.c
new file mode 100644
index 0000000..13fb676
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_7.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:04 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 7 -name r2hcII_7 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 24 FP additions, 18 FP multiplications,
+ * (or, 12 additions, 6 multiplications, 12 fused multiply/add),
+ * 20 stack variables, and 14 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_7.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_7.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_7.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_7(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP900968867, +0.900968867902419126236102319507445051165919162);
+     DK(KP222520933, +0.222520933956314404288902564496794759466355569);
+     DK(KP623489801, +0.623489801858733530525004884004239810632274731);
+     DK(KP433883739, +0.433883739117558120475768332848358754609990728);
+     DK(KP974927912, +0.974927912181823607018131682993931217232785801);
+     DK(KP781831482, +0.781831482468029808708444526674057750232334519);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, Ta, Td, T4, Tb, T7, Tc, T8, T9;
+	  T1 = I[0];
+	  T8 = I[WS(is, 1)];
+	  T9 = I[WS(is, 6)];
+	  Ta = T8 - T9;
+	  Td = T8 + T9;
+	  {
+	       E T2, T3, T5, T6;
+	       T2 = I[WS(is, 2)];
+	       T3 = I[WS(is, 5)];
+	       T4 = T2 - T3;
+	       Tb = T2 + T3;
+	       T5 = I[WS(is, 3)];
+	       T6 = I[WS(is, 4)];
+	       T7 = T5 - T6;
+	       Tc = T5 + T6;
+	  }
+	  io[0] = -(FMA(KP781831482, Tb, KP974927912 * Tc) + (KP433883739 * Td));
+	  io[WS(ios, 1)] = FNMS(KP974927912, Td, KP781831482 * Tc) - (KP433883739 * Tb);
+	  ro[0] = FMA(KP623489801, T4, T1) + FMA(KP222520933, T7, KP900968867 * Ta);
+	  io[WS(ios, 2)] = FNMS(KP781831482, Td, KP974927912 * Tb) - (KP433883739 * Tc);
+	  ro[WS(ros, 2)] = FMA(KP900968867, T7, T1) + FNMA(KP623489801, Ta, KP222520933 * T4);
+	  ro[WS(ros, 1)] = FMA(KP222520933, Ta, T1) + FNMA(KP623489801, T7, KP900968867 * T4);
+	  ro[WS(ros, 3)] = T1 + T4 - (T7 + Ta);
+     }
+}
+
+static const kr2hc_desc desc = { 7, "r2hcII_7", {12, 6, 12, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_7) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_7, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_8.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_8.c
new file mode 100644
index 0000000..d6d2aaf
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_8.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:04 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 8 -name r2hcII_8 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 22 FP additions, 10 FP multiplications,
+ * (or, 18 additions, 6 multiplications, 4 fused multiply/add),
+ * 18 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_8(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, Tj, T4, Ti, T8, Te, Tb, Tf, T2, T3;
+	  T1 = I[0];
+	  Tj = I[WS(is, 4)];
+	  T2 = I[WS(is, 2)];
+	  T3 = I[WS(is, 6)];
+	  T4 = KP707106781 * (T2 - T3);
+	  Ti = KP707106781 * (T2 + T3);
+	  {
+	       E T6, T7, T9, Ta;
+	       T6 = I[WS(is, 1)];
+	       T7 = I[WS(is, 5)];
+	       T8 = FNMS(KP382683432, T7, KP923879532 * T6);
+	       Te = FMA(KP382683432, T6, KP923879532 * T7);
+	       T9 = I[WS(is, 3)];
+	       Ta = I[WS(is, 7)];
+	       Tb = FNMS(KP923879532, Ta, KP382683432 * T9);
+	       Tf = FMA(KP923879532, T9, KP382683432 * Ta);
+	  }
+	  {
+	       E T5, Tc, Th, Tk;
+	       T5 = T1 + T4;
+	       Tc = T8 + Tb;
+	       ro[WS(ros, 3)] = T5 - Tc;
+	       ro[0] = T5 + Tc;
+	       Th = Te + Tf;
+	       Tk = Ti + Tj;
+	       io[0] = -(Th + Tk);
+	       io[WS(ios, 3)] = Tk - Th;
+	  }
+	  {
+	       E Td, Tg, Tl, Tm;
+	       Td = T1 - T4;
+	       Tg = Te - Tf;
+	       ro[WS(ros, 2)] = Td - Tg;
+	       ro[WS(ros, 1)] = Td + Tg;
+	       Tl = Tb - T8;
+	       Tm = Tj - Ti;
+	       io[WS(ios, 2)] = Tl - Tm;
+	       io[WS(ios, 1)] = Tl + Tm;
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 8, "r2hcII_8", {18, 6, 4, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_8) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_8, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hcII_9.c b/src/fftw3/rdft/codelets/r2hc/r2hcII_9.c
new file mode 100644
index 0000000..a046b93
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hcII_9.c
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:58:09 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 9 -name r2hcII_9 -dft-II -include r2hcII.h */
+
+/*
+ * This function contains 42 FP additions, 30 FP multiplications,
+ * (or, 25 additions, 13 multiplications, 17 fused multiply/add),
+ * 39 stack variables, and 18 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hcII_9.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_9.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hcII_9.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hcII.h"
+
+static void r2hcII_9(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP663413948, +0.663413948168938396205421319635891297216863310);
+     DK(KP642787609, +0.642787609686539326322643409907263432907559884);
+     DK(KP556670399, +0.556670399226419366452912952047023132968291906);
+     DK(KP766044443, +0.766044443118978035202392650555416673935832457);
+     DK(KP852868531, +0.852868531952443209628250963940074071936020296);
+     DK(KP173648177, +0.173648177666930348851716626769314796000375677);
+     DK(KP984807753, +0.984807753012208059366743024589523013670643252);
+     DK(KP150383733, +0.150383733180435296639271897612501926072238258);
+     DK(KP813797681, +0.813797681349373692844693217248393223289101568);
+     DK(KP342020143, +0.342020143325668733044099614682259580763083368);
+     DK(KP939692620, +0.939692620785908384054109277324731469936208134);
+     DK(KP296198132, +0.296198132726023843175338011893050938967728390);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T4, To, Ta, Tl, Tk, Tf, Ti, Th, T2, T3, T5, Tg;
+	  T1 = I[0];
+	  T2 = I[WS(is, 3)];
+	  T3 = I[WS(is, 6)];
+	  T4 = T2 - T3;
+	  To = T2 + T3;
+	  {
+	       E T6, T7, T8, T9;
+	       T6 = I[WS(is, 2)];
+	       T7 = I[WS(is, 5)];
+	       T8 = I[WS(is, 8)];
+	       T9 = T7 - T8;
+	       Ta = T6 - T9;
+	       Tl = T7 + T8;
+	       Tk = FMA(KP500000000, T9, T6);
+	  }
+	  {
+	       E Tb, Tc, Td, Te;
+	       Tb = I[WS(is, 4)];
+	       Tc = I[WS(is, 1)];
+	       Td = I[WS(is, 7)];
+	       Te = Tc + Td;
+	       Tf = Tb - Te;
+	       Ti = FMA(KP500000000, Te, Tb);
+	       Th = Tc - Td;
+	  }
+	  io[WS(ios, 1)] = KP866025403 * (Tf - Ta);
+	  T5 = T1 - T4;
+	  Tg = Ta + Tf;
+	  ro[WS(ros, 1)] = FNMS(KP500000000, Tg, T5);
+	  ro[WS(ros, 4)] = T5 + Tg;
+	  {
+	       E Tr, Tt, Tw, Tv, Tu, Tp, Tq, Ts, Tj, Tm, Tn;
+	       Tr = FMA(KP500000000, T4, T1);
+	       Tt = FMA(KP296198132, Th, KP939692620 * Ti);
+	       Tw = FNMS(KP813797681, Th, KP342020143 * Ti);
+	       Tv = FNMS(KP984807753, Tk, KP150383733 * Tl);
+	       Tu = FMA(KP173648177, Tk, KP852868531 * Tl);
+	       Tp = FNMS(KP556670399, Tl, KP766044443 * Tk);
+	       Tq = FMA(KP852868531, Th, KP173648177 * Ti);
+	       Ts = Tp + Tq;
+	       Tj = FNMS(KP984807753, Ti, KP150383733 * Th);
+	       Tm = FMA(KP642787609, Tk, KP663413948 * Tl);
+	       Tn = Tj - Tm;
+	       io[0] = FNMS(KP866025403, To, Tn);
+	       ro[0] = Tr + Ts;
+	       io[WS(ios, 3)] = FNMS(KP500000000, Tn, KP866025403 * ((Tp - Tq) - To));
+	       ro[WS(ros, 3)] = FMA(KP866025403, Tm + Tj, Tr) - (KP500000000 * Ts);
+	       io[WS(ios, 2)] = FMA(KP866025403, To - (Tu + Tt), KP500000000 * (Tw - Tv));
+	       ro[WS(ros, 2)] = FMA(KP500000000, Tt - Tu, Tr) + (KP866025403 * (Tv + Tw));
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 9, "r2hcII_9", {25, 13, 17, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hcII_9) (planner *p) {
+     X(kr2hcII_register) (p, r2hcII_9, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_10.c b/src/fftw3/rdft/codelets/r2hc/r2hc_10.c
new file mode 100644
index 0000000..38e1766
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_10.c
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:41 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 10 -name r2hc_10 -include r2hc.h */
+
+/*
+ * This function contains 34 FP additions, 12 FP multiplications,
+ * (or, 28 additions, 6 multiplications, 6 fused multiply/add),
+ * 26 stack variables, and 20 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_10.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_10.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_10.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_10(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E Ti, Tt, Ta, Tn, Td, To, Te, Tv, T3, Tq, T6, Tr, T7, Tu, Tg;
+	  E Th;
+	  Tg = I[0];
+	  Th = I[WS(is, 5)];
+	  Ti = Tg - Th;
+	  Tt = Tg + Th;
+	  {
+	       E T8, T9, Tb, Tc;
+	       T8 = I[WS(is, 4)];
+	       T9 = I[WS(is, 9)];
+	       Ta = T8 - T9;
+	       Tn = T8 + T9;
+	       Tb = I[WS(is, 6)];
+	       Tc = I[WS(is, 1)];
+	       Td = Tb - Tc;
+	       To = Tb + Tc;
+	  }
+	  Te = Ta + Td;
+	  Tv = Tn + To;
+	  {
+	       E T1, T2, T4, T5;
+	       T1 = I[WS(is, 2)];
+	       T2 = I[WS(is, 7)];
+	       T3 = T1 - T2;
+	       Tq = T1 + T2;
+	       T4 = I[WS(is, 8)];
+	       T5 = I[WS(is, 3)];
+	       T6 = T4 - T5;
+	       Tr = T4 + T5;
+	  }
+	  T7 = T3 + T6;
+	  Tu = Tq + Tr;
+	  {
+	       E Tl, Tm, Tf, Tj, Tk;
+	       Tl = Td - Ta;
+	       Tm = T3 - T6;
+	       io[WS(ios, 1)] = FNMS(KP951056516, Tm, KP587785252 * Tl);
+	       io[WS(ios, 3)] = FMA(KP587785252, Tm, KP951056516 * Tl);
+	       Tf = KP559016994 * (T7 - Te);
+	       Tj = T7 + Te;
+	       Tk = FNMS(KP250000000, Tj, Ti);
+	       ro[WS(ros, 1)] = Tf + Tk;
+	       ro[WS(ros, 5)] = Ti + Tj;
+	       ro[WS(ros, 3)] = Tk - Tf;
+	  }
+	  {
+	       E Tp, Ts, Ty, Tw, Tx;
+	       Tp = Tn - To;
+	       Ts = Tq - Tr;
+	       io[WS(ios, 2)] = FNMS(KP587785252, Ts, KP951056516 * Tp);
+	       io[WS(ios, 4)] = FMA(KP951056516, Ts, KP587785252 * Tp);
+	       Ty = KP559016994 * (Tu - Tv);
+	       Tw = Tu + Tv;
+	       Tx = FNMS(KP250000000, Tw, Tt);
+	       ro[WS(ros, 2)] = Tx - Ty;
+	       ro[0] = Tt + Tw;
+	       ro[WS(ros, 4)] = Ty + Tx;
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 10, "r2hc_10", {28, 6, 6, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_10) (planner *p) {
+     X(kr2hc_register) (p, r2hc_10, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_11.c b/src/fftw3/rdft/codelets/r2hc/r2hc_11.c
new file mode 100644
index 0000000..cceb091
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_11.c
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:41 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 11 -name r2hc_11 -include r2hc.h */
+
+/*
+ * This function contains 60 FP additions, 50 FP multiplications,
+ * (or, 20 additions, 10 multiplications, 40 fused multiply/add),
+ * 28 stack variables, and 22 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_11.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_11.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_11.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_11(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP654860733, +0.654860733945285064056925072466293553183791199);
+     DK(KP142314838, +0.142314838273285140443792668616369668791051361);
+     DK(KP959492973, +0.959492973614497389890368057066327699062454848);
+     DK(KP415415013, +0.415415013001886425529274149229623203524004910);
+     DK(KP841253532, +0.841253532831181168861811648919367717513292498);
+     DK(KP989821441, +0.989821441880932732376092037776718787376519372);
+     DK(KP909631995, +0.909631995354518371411715383079028460060241051);
+     DK(KP281732556, +0.281732556841429697711417915346616899035777899);
+     DK(KP540640817, +0.540640817455597582107635954318691695431770608);
+     DK(KP755749574, +0.755749574354258283774035843972344420179717445);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T4, Tl, Tg, Th, Td, Ti, Ta, Tk, T7, Tj, Tb, Tc;
+	  T1 = I[0];
+	  {
+	       E T2, T3, Te, Tf;
+	       T2 = I[WS(is, 2)];
+	       T3 = I[WS(is, 9)];
+	       T4 = T2 + T3;
+	       Tl = T3 - T2;
+	       Te = I[WS(is, 1)];
+	       Tf = I[WS(is, 10)];
+	       Tg = Te + Tf;
+	       Th = Tf - Te;
+	  }
+	  Tb = I[WS(is, 3)];
+	  Tc = I[WS(is, 8)];
+	  Td = Tb + Tc;
+	  Ti = Tc - Tb;
+	  {
+	       E T8, T9, T5, T6;
+	       T8 = I[WS(is, 5)];
+	       T9 = I[WS(is, 6)];
+	       Ta = T8 + T9;
+	       Tk = T9 - T8;
+	       T5 = I[WS(is, 4)];
+	       T6 = I[WS(is, 7)];
+	       T7 = T5 + T6;
+	       Tj = T6 - T5;
+	  }
+	  io[WS(ios, 4)] = FMA(KP755749574, Th, KP540640817 * Ti) + FNMS(KP909631995, Tk, KP281732556 * Tj) - (KP989821441 * Tl);
+	  ro[WS(ros, 4)] = FMA(KP841253532, Td, T1) + FNMS(KP959492973, T7, KP415415013 * Ta) + FNMA(KP142314838, T4, KP654860733 * Tg);
+	  io[WS(ios, 2)] = FMA(KP909631995, Th, KP755749574 * Tl) + FNMA(KP540640817, Tk, KP989821441 * Tj) - (KP281732556 * Ti);
+	  io[WS(ios, 5)] = FMA(KP281732556, Th, KP755749574 * Ti) + FNMS(KP909631995, Tj, KP989821441 * Tk) - (KP540640817 * Tl);
+	  io[WS(ios, 1)] = FMA(KP540640817, Th, KP909631995 * Tl) + FMA(KP989821441, Ti, KP755749574 * Tj) + (KP281732556 * Tk);
+	  io[WS(ios, 3)] = FMA(KP989821441, Th, KP540640817 * Tj) + FNMS(KP909631995, Ti, KP755749574 * Tk) - (KP281732556 * Tl);
+	  ro[WS(ros, 3)] = FMA(KP415415013, Td, T1) + FNMS(KP654860733, Ta, KP841253532 * T7) + FNMA(KP959492973, T4, KP142314838 * Tg);
+	  ro[WS(ros, 1)] = FMA(KP841253532, Tg, T1) + FNMS(KP959492973, Ta, KP415415013 * T4) + FNMA(KP654860733, T7, KP142314838 * Td);
+	  ro[0] = T1 + Tg + T4 + Td + T7 + Ta;
+	  ro[WS(ros, 2)] = FMA(KP415415013, Tg, T1) + FNMS(KP142314838, T7, KP841253532 * Ta) + FNMA(KP959492973, Td, KP654860733 * T4);
+	  ro[WS(ros, 5)] = FMA(KP841253532, T4, T1) + FNMS(KP142314838, Ta, KP415415013 * T7) + FNMA(KP654860733, Td, KP959492973 * Tg);
+     }
+}
+
+static const kr2hc_desc desc = { 11, "r2hc_11", {20, 10, 40, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_11) (planner *p) {
+     X(kr2hc_register) (p, r2hc_11, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_12.c b/src/fftw3/rdft/codelets/r2hc/r2hc_12.c
new file mode 100644
index 0000000..81a5273
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_12.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:41 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 12 -name r2hc_12 -include r2hc.h */
+
+/*
+ * This function contains 38 FP additions, 8 FP multiplications,
+ * (or, 34 additions, 4 multiplications, 4 fused multiply/add),
+ * 21 stack variables, and 24 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_12.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_12.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_12.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_12(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T5, Tp, Tb, Tn, Ty, Tt, Ta, Tq, Tc, Ti, Tz, Tu, Td, To;
+	  {
+	       E T1, T2, T3, T4;
+	       T1 = I[0];
+	       T2 = I[WS(is, 4)];
+	       T3 = I[WS(is, 8)];
+	       T4 = T2 + T3;
+	       T5 = T1 + T4;
+	       Tp = FNMS(KP500000000, T4, T1);
+	       Tb = T3 - T2;
+	  }
+	  {
+	       E Tj, Tk, Tl, Tm;
+	       Tj = I[WS(is, 3)];
+	       Tk = I[WS(is, 7)];
+	       Tl = I[WS(is, 11)];
+	       Tm = Tk + Tl;
+	       Tn = FNMS(KP500000000, Tm, Tj);
+	       Ty = Tl - Tk;
+	       Tt = Tj + Tm;
+	  }
+	  {
+	       E T6, T7, T8, T9;
+	       T6 = I[WS(is, 6)];
+	       T7 = I[WS(is, 10)];
+	       T8 = I[WS(is, 2)];
+	       T9 = T7 + T8;
+	       Ta = T6 + T9;
+	       Tq = FNMS(KP500000000, T9, T6);
+	       Tc = T8 - T7;
+	  }
+	  {
+	       E Te, Tf, Tg, Th;
+	       Te = I[WS(is, 9)];
+	       Tf = I[WS(is, 1)];
+	       Tg = I[WS(is, 5)];
+	       Th = Tf + Tg;
+	       Ti = FNMS(KP500000000, Th, Te);
+	       Tz = Tg - Tf;
+	       Tu = Te + Th;
+	  }
+	  ro[WS(ros, 3)] = T5 - Ta;
+	  io[WS(ios, 3)] = Tt - Tu;
+	  Td = KP866025403 * (Tb - Tc);
+	  To = Ti - Tn;
+	  io[WS(ios, 1)] = Td + To;
+	  io[WS(ios, 5)] = To - Td;
+	  {
+	       E Tx, TA, Tv, Tw;
+	       Tx = Tp - Tq;
+	       TA = KP866025403 * (Ty - Tz);
+	       ro[WS(ros, 5)] = Tx - TA;
+	       ro[WS(ros, 1)] = Tx + TA;
+	       Tv = T5 + Ta;
+	       Tw = Tt + Tu;
+	       ro[WS(ros, 6)] = Tv - Tw;
+	       ro[0] = Tv + Tw;
+	  }
+	  {
+	       E Tr, Ts, TB, TC;
+	       Tr = Tp + Tq;
+	       Ts = Tn + Ti;
+	       ro[WS(ros, 2)] = Tr - Ts;
+	       ro[WS(ros, 4)] = Tr + Ts;
+	       TB = Ty + Tz;
+	       TC = Tb + Tc;
+	       io[WS(ios, 2)] = KP866025403 * (TB - TC);
+	       io[WS(ios, 4)] = KP866025403 * (TC + TB);
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 12, "r2hc_12", {34, 4, 4, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_12) (planner *p) {
+     X(kr2hc_register) (p, r2hc_12, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_13.c b/src/fftw3/rdft/codelets/r2hc/r2hc_13.c
new file mode 100644
index 0000000..deb7914
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_13.c
@@ -0,0 +1,194 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:41 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 13 -name r2hc_13 -include r2hc.h */
+
+/*
+ * This function contains 76 FP additions, 34 FP multiplications,
+ * (or, 57 additions, 15 multiplications, 19 fused multiply/add),
+ * 55 stack variables, and 26 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_13.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_13.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_13.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_13(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP083333333, +0.083333333333333333333333333333333333333333333);
+     DK(KP075902986, +0.075902986037193865983102897245103540356428373);
+     DK(KP251768516, +0.251768516431883313623436926934233488546674281);
+     DK(KP503537032, +0.503537032863766627246873853868466977093348562);
+     DK(KP113854479, +0.113854479055790798974654345867655310534642560);
+     DK(KP265966249, +0.265966249214837287587521063842185948798330267);
+     DK(KP387390585, +0.387390585467617292130675966426762851778775217);
+     DK(KP300462606, +0.300462606288665774426601772289207995520941381);
+     DK(KP132983124, +0.132983124607418643793760531921092974399165133);
+     DK(KP258260390, +0.258260390311744861420450644284508567852516811);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_732050807, +1.732050807568877293527446341505872366942805254);
+     DK(KP300238635, +0.300238635966332641462884626667381504676006424);
+     DK(KP011599105, +0.011599105605768290721655456654083252189827041);
+     DK(KP156891391, +0.156891391051584611046832726756003269660212636);
+     DK(KP256247671, +0.256247671582936600958684654061725059144125175);
+     DK(KP174138601, +0.174138601152135905005660794929264742616964676);
+     DK(KP575140729, +0.575140729474003121368385547455453388461001608);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T13, Tb, Tm, TW, TX, T14, TU, T10, Tz, TB, Tu, TC, TR, T11;
+	  T13 = I[0];
+	  {
+	       E Te, TO, Ta, Tv, To, T5, Tw, Tp, Th, Tr, Tk, Ts, Tl, TP, Tc;
+	       E Td;
+	       Tc = I[WS(is, 8)];
+	       Td = I[WS(is, 5)];
+	       Te = Tc - Td;
+	       TO = Tc + Td;
+	       {
+		    E T6, T7, T8, T9;
+		    T6 = I[WS(is, 1)];
+		    T7 = I[WS(is, 3)];
+		    T8 = I[WS(is, 9)];
+		    T9 = T7 + T8;
+		    Ta = T6 + T9;
+		    Tv = T7 - T8;
+		    To = FNMS(KP500000000, T9, T6);
+	       }
+	       {
+		    E T1, T2, T3, T4;
+		    T1 = I[WS(is, 12)];
+		    T2 = I[WS(is, 10)];
+		    T3 = I[WS(is, 4)];
+		    T4 = T2 + T3;
+		    T5 = T1 + T4;
+		    Tw = T2 - T3;
+		    Tp = FNMS(KP500000000, T4, T1);
+	       }
+	       {
+		    E Tf, Tg, Ti, Tj;
+		    Tf = I[WS(is, 11)];
+		    Tg = I[WS(is, 6)];
+		    Th = Tf - Tg;
+		    Tr = Tf + Tg;
+		    Ti = I[WS(is, 7)];
+		    Tj = I[WS(is, 2)];
+		    Tk = Ti - Tj;
+		    Ts = Ti + Tj;
+	       }
+	       Tl = Th + Tk;
+	       TP = Tr + Ts;
+	       Tb = T5 - Ta;
+	       Tm = Te + Tl;
+	       TW = Ta + T5;
+	       TX = TO + TP;
+	       T14 = TW + TX;
+	       {
+		    E TS, TT, Tx, Ty;
+		    TS = Tv + Tw;
+		    TT = Th - Tk;
+		    TU = TS - TT;
+		    T10 = TS + TT;
+		    Tx = KP866025403 * (Tv - Tw);
+		    Ty = FNMS(KP500000000, Tl, Te);
+		    Tz = Tx + Ty;
+		    TB = Ty - Tx;
+	       }
+	       {
+		    E Tq, Tt, TN, TQ;
+		    Tq = To - Tp;
+		    Tt = KP866025403 * (Tr - Ts);
+		    Tu = Tq - Tt;
+		    TC = Tq + Tt;
+		    TN = To + Tp;
+		    TQ = FNMS(KP500000000, TP, TO);
+		    TR = TN - TQ;
+		    T11 = TN + TQ;
+	       }
+	  }
+	  ro[0] = T13 + T14;
+	  {
+	       E Tn, TG, TE, TF, TJ, TM, TK, TL;
+	       Tn = FNMS(KP174138601, Tm, KP575140729 * Tb);
+	       TG = FMA(KP174138601, Tb, KP575140729 * Tm);
+	       {
+		    E TA, TD, TH, TI;
+		    TA = FNMS(KP156891391, Tz, KP256247671 * Tu);
+		    TD = FNMS(KP300238635, TC, KP011599105 * TB);
+		    TE = TA + TD;
+		    TF = KP1_732050807 * (TD - TA);
+		    TH = FMA(KP300238635, TB, KP011599105 * TC);
+		    TI = FMA(KP256247671, Tz, KP156891391 * Tu);
+		    TJ = TH - TI;
+		    TM = KP1_732050807 * (TI + TH);
+	       }
+	       io[WS(ios, 5)] = FMA(KP2_000000000, TE, Tn);
+	       io[WS(ios, 1)] = FMA(KP2_000000000, TJ, TG);
+	       TK = TG - TJ;
+	       io[WS(ios, 4)] = TF - TK;
+	       io[WS(ios, 3)] = TF + TK;
+	       TL = Tn - TE;
+	       io[WS(ios, 2)] = TL - TM;
+	       io[WS(ios, 6)] = TL + TM;
+	  }
+	  {
+	       E TZ, T1b, T19, T1e, T16, T1a, TV, TY, T1c, T1d;
+	       TV = FNMS(KP132983124, TU, KP258260390 * TR);
+	       TY = KP300462606 * (TW - TX);
+	       TZ = FMA(KP2_000000000, TV, TY);
+	       T1b = TY - TV;
+	       {
+		    E T17, T18, T12, T15;
+		    T17 = FMA(KP387390585, TU, KP265966249 * TR);
+		    T18 = FNMS(KP503537032, T11, KP113854479 * T10);
+		    T19 = T17 - T18;
+		    T1e = T17 + T18;
+		    T12 = FMA(KP251768516, T10, KP075902986 * T11);
+		    T15 = FNMS(KP083333333, T14, T13);
+		    T16 = FMA(KP2_000000000, T12, T15);
+		    T1a = T15 - T12;
+	       }
+	       ro[WS(ros, 1)] = TZ + T16;
+	       ro[WS(ros, 5)] = T16 - TZ;
+	       T1c = T1a - T1b;
+	       ro[WS(ros, 2)] = T19 + T1c;
+	       ro[WS(ros, 6)] = T1c - T19;
+	       T1d = T1b + T1a;
+	       ro[WS(ros, 3)] = T1d - T1e;
+	       ro[WS(ros, 4)] = T1e + T1d;
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 13, "r2hc_13", {57, 15, 19, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_13) (planner *p) {
+     X(kr2hc_register) (p, r2hc_13, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_14.c b/src/fftw3/rdft/codelets/r2hc/r2hc_14.c
new file mode 100644
index 0000000..497d84a
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_14.c
@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:41 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 14 -name r2hc_14 -include r2hc.h */
+
+/*
+ * This function contains 62 FP additions, 36 FP multiplications,
+ * (or, 38 additions, 12 multiplications, 24 fused multiply/add),
+ * 29 stack variables, and 28 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_14.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_14.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_14.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_14(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP900968867, +0.900968867902419126236102319507445051165919162);
+     DK(KP222520933, +0.222520933956314404288902564496794759466355569);
+     DK(KP623489801, +0.623489801858733530525004884004239810632274731);
+     DK(KP433883739, +0.433883739117558120475768332848358754609990728);
+     DK(KP974927912, +0.974927912181823607018131682993931217232785801);
+     DK(KP781831482, +0.781831482468029808708444526674057750232334519);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T3, TB, T6, Tv, Tn, Ts, Tk, Tt, Td, Ty, T9, Tw, Tg, Tz, T1;
+	  E T2;
+	  T1 = I[0];
+	  T2 = I[WS(is, 7)];
+	  T3 = T1 - T2;
+	  TB = T1 + T2;
+	  {
+	       E T4, T5, Tl, Tm;
+	       T4 = I[WS(is, 4)];
+	       T5 = I[WS(is, 11)];
+	       T6 = T4 - T5;
+	       Tv = T4 + T5;
+	       Tl = I[WS(is, 12)];
+	       Tm = I[WS(is, 5)];
+	       Tn = Tl - Tm;
+	       Ts = Tl + Tm;
+	  }
+	  {
+	       E Ti, Tj, Tb, Tc;
+	       Ti = I[WS(is, 2)];
+	       Tj = I[WS(is, 9)];
+	       Tk = Ti - Tj;
+	       Tt = Ti + Tj;
+	       Tb = I[WS(is, 6)];
+	       Tc = I[WS(is, 13)];
+	       Td = Tb - Tc;
+	       Ty = Tb + Tc;
+	  }
+	  {
+	       E T7, T8, Te, Tf;
+	       T7 = I[WS(is, 10)];
+	       T8 = I[WS(is, 3)];
+	       T9 = T7 - T8;
+	       Tw = T7 + T8;
+	       Te = I[WS(is, 8)];
+	       Tf = I[WS(is, 1)];
+	       Tg = Te - Tf;
+	       Tz = Te + Tf;
+	  }
+	  {
+	       E Tp, Tr, Tq, Ta, To, Th;
+	       Tp = Tn - Tk;
+	       Tr = Tg - Td;
+	       Tq = T9 - T6;
+	       io[WS(ios, 1)] = FMA(KP781831482, Tp, KP974927912 * Tq) + (KP433883739 * Tr);
+	       io[WS(ios, 5)] = FMA(KP433883739, Tq, KP781831482 * Tr) - (KP974927912 * Tp);
+	       io[WS(ios, 3)] = FMA(KP433883739, Tp, KP974927912 * Tr) - (KP781831482 * Tq);
+	       Ta = T6 + T9;
+	       To = Tk + Tn;
+	       Th = Td + Tg;
+	       ro[WS(ros, 3)] = FMA(KP623489801, Ta, T3) + FNMA(KP222520933, Th, KP900968867 * To);
+	       ro[WS(ros, 7)] = T3 + To + Ta + Th;
+	       ro[WS(ros, 1)] = FMA(KP623489801, To, T3) + FNMA(KP900968867, Th, KP222520933 * Ta);
+	       ro[WS(ros, 5)] = FMA(KP623489801, Th, T3) + FNMA(KP900968867, Ta, KP222520933 * To);
+	  }
+	  {
+	       E Tu, TA, Tx, TC, TE, TD;
+	       Tu = Ts - Tt;
+	       TA = Ty - Tz;
+	       Tx = Tv - Tw;
+	       io[WS(ios, 2)] = FMA(KP974927912, Tu, KP433883739 * Tx) + (KP781831482 * TA);
+	       io[WS(ios, 6)] = FMA(KP974927912, Tx, KP433883739 * TA) - (KP781831482 * Tu);
+	       io[WS(ios, 4)] = FNMS(KP781831482, Tx, KP974927912 * TA) - (KP433883739 * Tu);
+	       TC = Tt + Ts;
+	       TE = Tv + Tw;
+	       TD = Ty + Tz;
+	       ro[WS(ros, 6)] = FMA(KP623489801, TC, TB) + FNMA(KP900968867, TD, KP222520933 * TE);
+	       ro[WS(ros, 2)] = FMA(KP623489801, TD, TB) + FNMA(KP900968867, TE, KP222520933 * TC);
+	       ro[WS(ros, 4)] = FMA(KP623489801, TE, TB) + FNMA(KP222520933, TD, KP900968867 * TC);
+	       ro[0] = TB + TC + TE + TD;
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 14, "r2hc_14", {38, 12, 24, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_14) (planner *p) {
+     X(kr2hc_register) (p, r2hc_14, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_15.c b/src/fftw3/rdft/codelets/r2hc/r2hc_15.c
new file mode 100644
index 0000000..71c4aba
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_15.c
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:42 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 15 -name r2hc_15 -include r2hc.h */
+
+/*
+ * This function contains 64 FP additions, 25 FP multiplications,
+ * (or, 50 additions, 11 multiplications, 14 fused multiply/add),
+ * 47 stack variables, and 30 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_15.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_15.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_15.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_15(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP484122918, +0.484122918275927110647408174972799951354115213);
+     DK(KP216506350, +0.216506350946109661690930792688234045867850657);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP509036960, +0.509036960455127183450980863393907648510733164);
+     DK(KP823639103, +0.823639103546331925877420039278190003029660514);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E Ti, TR, TL, TD, TE, T7, Te, Tf, TV, TW, TX, Tv, Ty, TH, To;
+	  E Tr, TG, TS, TT, TU;
+	  {
+	       E TJ, Tg, Th, TK;
+	       TJ = I[0];
+	       Tg = I[WS(is, 10)];
+	       Th = I[WS(is, 5)];
+	       TK = Th + Tg;
+	       Ti = Tg - Th;
+	       TR = TJ + TK;
+	       TL = FNMS(KP500000000, TK, TJ);
+	  }
+	  {
+	       E Tm, Tt, Tw, Tp, T3, Tx, Ta, Tn, Td, Tq, T6, Tu;
+	       Tm = I[WS(is, 3)];
+	       Tt = I[WS(is, 6)];
+	       Tw = I[WS(is, 9)];
+	       Tp = I[WS(is, 12)];
+	       {
+		    E T1, T2, T8, T9;
+		    T1 = I[WS(is, 14)];
+		    T2 = I[WS(is, 4)];
+		    T3 = T1 - T2;
+		    Tx = T1 + T2;
+		    T8 = I[WS(is, 13)];
+		    T9 = I[WS(is, 8)];
+		    Ta = T8 - T9;
+		    Tn = T9 + T8;
+	       }
+	       {
+		    E Tb, Tc, T4, T5;
+		    Tb = I[WS(is, 7)];
+		    Tc = I[WS(is, 2)];
+		    Td = Tb - Tc;
+		    Tq = Tc + Tb;
+		    T4 = I[WS(is, 1)];
+		    T5 = I[WS(is, 11)];
+		    T6 = T4 - T5;
+		    Tu = T5 + T4;
+	       }
+	       TD = Ta - Td;
+	       TE = T6 + T3;
+	       T7 = T3 - T6;
+	       Te = Ta + Td;
+	       Tf = T7 - Te;
+	       TV = Tt + Tu;
+	       TW = Tw + Tx;
+	       TX = TV + TW;
+	       Tv = FNMS(KP500000000, Tu, Tt);
+	       Ty = FNMS(KP500000000, Tx, Tw);
+	       TH = Tv + Ty;
+	       To = FNMS(KP500000000, Tn, Tm);
+	       Tr = FNMS(KP500000000, Tq, Tp);
+	       TG = To + Tr;
+	       TS = Tm + Tn;
+	       TT = Tp + Tq;
+	       TU = TS + TT;
+	  }
+	  io[WS(ios, 5)] = KP866025403 * (Tf - Ti);
+	  {
+	       E TF, TP, TI, TM, TN, TQ, TO;
+	       TF = FMA(KP823639103, TD, KP509036960 * TE);
+	       TP = FNMS(KP509036960, TD, KP823639103 * TE);
+	       TI = KP559016994 * (TG - TH);
+	       TM = TG + TH;
+	       TN = FNMS(KP250000000, TM, TL);
+	       ro[WS(ros, 5)] = TL + TM;
+	       TQ = TN - TI;
+	       ro[WS(ros, 2)] = TP + TQ;
+	       ro[WS(ros, 7)] = TQ - TP;
+	       TO = TI + TN;
+	       ro[WS(ros, 1)] = TF + TO;
+	       ro[WS(ros, 4)] = TO - TF;
+	  }
+	  {
+	       E T11, T12, T10, TY, TZ;
+	       T11 = TS - TT;
+	       T12 = TW - TV;
+	       io[WS(ios, 3)] = FMA(KP587785252, T11, KP951056516 * T12);
+	       io[WS(ios, 6)] = FNMS(KP951056516, T11, KP587785252 * T12);
+	       T10 = KP559016994 * (TU - TX);
+	       TY = TU + TX;
+	       TZ = FNMS(KP250000000, TY, TR);
+	       ro[WS(ros, 3)] = TZ - T10;
+	       ro[0] = TR + TY;
+	       ro[WS(ros, 6)] = T10 + TZ;
+	       {
+		    E Tl, TB, TA, TC;
+		    {
+			 E Tj, Tk, Ts, Tz;
+			 Tj = FMA(KP866025403, Ti, KP216506350 * Tf);
+			 Tk = KP484122918 * (Te + T7);
+			 Tl = Tj + Tk;
+			 TB = Tk - Tj;
+			 Ts = To - Tr;
+			 Tz = Tv - Ty;
+			 TA = FMA(KP951056516, Ts, KP587785252 * Tz);
+			 TC = FNMS(KP587785252, Ts, KP951056516 * Tz);
+		    }
+		    io[WS(ios, 1)] = Tl - TA;
+		    io[WS(ios, 7)] = TC - TB;
+		    io[WS(ios, 4)] = Tl + TA;
+		    io[WS(ios, 2)] = TB + TC;
+	       }
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 15, "r2hc_15", {50, 11, 14, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_15) (planner *p) {
+     X(kr2hc_register) (p, r2hc_15, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_16.c b/src/fftw3/rdft/codelets/r2hc/r2hc_16.c
new file mode 100644
index 0000000..e28d6d5
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_16.c
@@ -0,0 +1,159 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:43 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 16 -name r2hc_16 -include r2hc.h */
+
+/*
+ * This function contains 58 FP additions, 12 FP multiplications,
+ * (or, 54 additions, 8 multiplications, 4 fused multiply/add),
+ * 34 stack variables, and 32 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_16.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_16(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T3, T6, T7, Tz, Ti, Ta, Td, Te, TA, Th, Tq, TV, TF, TP, Tx;
+	  E TU, TE, TM, Tg, Tf, TJ, TQ;
+	  {
+	       E T1, T2, T4, T5;
+	       T1 = I[0];
+	       T2 = I[WS(is, 8)];
+	       T3 = T1 + T2;
+	       T4 = I[WS(is, 4)];
+	       T5 = I[WS(is, 12)];
+	       T6 = T4 + T5;
+	       T7 = T3 + T6;
+	       Tz = T1 - T2;
+	       Ti = T4 - T5;
+	  }
+	  {
+	       E T8, T9, Tb, Tc;
+	       T8 = I[WS(is, 2)];
+	       T9 = I[WS(is, 10)];
+	       Ta = T8 + T9;
+	       Tg = T8 - T9;
+	       Tb = I[WS(is, 14)];
+	       Tc = I[WS(is, 6)];
+	       Td = Tb + Tc;
+	       Tf = Tb - Tc;
+	  }
+	  Te = Ta + Td;
+	  TA = KP707106781 * (Tg + Tf);
+	  Th = KP707106781 * (Tf - Tg);
+	  {
+	       E Tm, TN, Tp, TO;
+	       {
+		    E Tk, Tl, Tn, To;
+		    Tk = I[WS(is, 15)];
+		    Tl = I[WS(is, 7)];
+		    Tm = Tk - Tl;
+		    TN = Tk + Tl;
+		    Tn = I[WS(is, 3)];
+		    To = I[WS(is, 11)];
+		    Tp = Tn - To;
+		    TO = Tn + To;
+	       }
+	       Tq = FNMS(KP923879532, Tp, KP382683432 * Tm);
+	       TV = TN + TO;
+	       TF = FMA(KP923879532, Tm, KP382683432 * Tp);
+	       TP = TN - TO;
+	  }
+	  {
+	       E Tt, TK, Tw, TL;
+	       {
+		    E Tr, Ts, Tu, Tv;
+		    Tr = I[WS(is, 1)];
+		    Ts = I[WS(is, 9)];
+		    Tt = Tr - Ts;
+		    TK = Tr + Ts;
+		    Tu = I[WS(is, 5)];
+		    Tv = I[WS(is, 13)];
+		    Tw = Tu - Tv;
+		    TL = Tu + Tv;
+	       }
+	       Tx = FMA(KP382683432, Tt, KP923879532 * Tw);
+	       TU = TK + TL;
+	       TE = FNMS(KP382683432, Tw, KP923879532 * Tt);
+	       TM = TK - TL;
+	  }
+	  ro[WS(ros, 4)] = T7 - Te;
+	  io[WS(ios, 4)] = TV - TU;
+	  {
+	       E Tj, Ty, TD, TG;
+	       Tj = Th - Ti;
+	       Ty = Tq - Tx;
+	       io[WS(ios, 1)] = Tj + Ty;
+	       io[WS(ios, 7)] = Ty - Tj;
+	       TD = Tz + TA;
+	       TG = TE + TF;
+	       ro[WS(ros, 7)] = TD - TG;
+	       ro[WS(ros, 1)] = TD + TG;
+	  }
+	  {
+	       E TB, TC, TH, TI;
+	       TB = Tz - TA;
+	       TC = Tx + Tq;
+	       ro[WS(ros, 5)] = TB - TC;
+	       ro[WS(ros, 3)] = TB + TC;
+	       TH = Ti + Th;
+	       TI = TF - TE;
+	       io[WS(ios, 3)] = TH + TI;
+	       io[WS(ios, 5)] = TI - TH;
+	  }
+	  TJ = T3 - T6;
+	  TQ = KP707106781 * (TM + TP);
+	  ro[WS(ros, 6)] = TJ - TQ;
+	  ro[WS(ros, 2)] = TJ + TQ;
+	  {
+	       E TR, TS, TT, TW;
+	       TR = Td - Ta;
+	       TS = KP707106781 * (TP - TM);
+	       io[WS(ios, 2)] = TR + TS;
+	       io[WS(ios, 6)] = TS - TR;
+	       TT = T7 + Te;
+	       TW = TU + TV;
+	       ro[WS(ros, 8)] = TT - TW;
+	       ro[0] = TT + TW;
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 16, "r2hc_16", {54, 8, 4, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_16) (planner *p) {
+     X(kr2hc_register) (p, r2hc_16, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_2.c b/src/fftw3/rdft/codelets/r2hc/r2hc_2.c
new file mode 100644
index 0000000..b0d8450
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_2.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:40 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 2 -name r2hc_2 -include r2hc.h */
+
+/*
+ * This function contains 2 FP additions, 0 FP multiplications,
+ * (or, 2 additions, 0 multiplications, 0 fused multiply/add),
+ * 3 stack variables, and 4 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_2.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_2.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_2.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_2(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T2;
+	  T1 = I[0];
+	  T2 = I[WS(is, 1)];
+	  ro[WS(ros, 1)] = T1 - T2;
+	  ro[0] = T1 + T2;
+     }
+}
+
+static const kr2hc_desc desc = { 2, "r2hc_2", {2, 0, 0, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_2) (planner *p) {
+     X(kr2hc_register) (p, r2hc_2, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_3.c b/src/fftw3/rdft/codelets/r2hc/r2hc_3.c
new file mode 100644
index 0000000..84041fc
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_3.c
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:40 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 3 -name r2hc_3 -include r2hc.h */
+
+/*
+ * This function contains 4 FP additions, 2 FP multiplications,
+ * (or, 3 additions, 1 multiplications, 1 fused multiply/add),
+ * 7 stack variables, and 6 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_3.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_3.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_3.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_3(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T2, T3, T4;
+	  T1 = I[0];
+	  T2 = I[WS(is, 1)];
+	  T3 = I[WS(is, 2)];
+	  T4 = T2 + T3;
+	  ro[WS(ros, 1)] = FNMS(KP500000000, T4, T1);
+	  io[WS(ios, 1)] = KP866025403 * (T3 - T2);
+	  ro[0] = T1 + T4;
+     }
+}
+
+static const kr2hc_desc desc = { 3, "r2hc_3", {3, 1, 1, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_3) (planner *p) {
+     X(kr2hc_register) (p, r2hc_3, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_32.c b/src/fftw3/rdft/codelets/r2hc/r2hc_32.c
new file mode 100644
index 0000000..b3e0a9d
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_32.c
@@ -0,0 +1,320 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:44 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 32 -name r2hc_32 -include r2hc.h */
+
+/*
+ * This function contains 156 FP additions, 42 FP multiplications,
+ * (or, 140 additions, 26 multiplications, 16 fused multiply/add),
+ * 54 stack variables, and 64 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_32.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_32(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP555570233, +0.555570233019602224742830813948532874374937191);
+     DK(KP831469612, +0.831469612302545237078788377617905756738560812);
+     DK(KP195090322, +0.195090322016128267848284868477022240927691618);
+     DK(KP980785280, +0.980785280403230449126182236134239036973933731);
+     DK(KP382683432, +0.382683432365089771728459984030398866761344562);
+     DK(KP923879532, +0.923879532511286756128183189396788286822416626);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T7, T2b, Tv, T1l, Te, T2o, Ty, T1k, Tt, T2d, TF, T1h, Tm, T2c, TC;
+	  E T1i, T1Z, T22, T2k, T2j, T1e, T1C, T19, T1B, T1S, T1V, T2h, T2g, TX, T1z;
+	  E TS, T1y;
+	  {
+	       E T1, T2, T3, T4, T5, T6;
+	       T1 = I[0];
+	       T2 = I[WS(is, 16)];
+	       T3 = T1 + T2;
+	       T4 = I[WS(is, 8)];
+	       T5 = I[WS(is, 24)];
+	       T6 = T4 + T5;
+	       T7 = T3 + T6;
+	       T2b = T3 - T6;
+	       Tv = T1 - T2;
+	       T1l = T4 - T5;
+	  }
+	  {
+	       E Ta, Tw, Td, Tx;
+	       {
+		    E T8, T9, Tb, Tc;
+		    T8 = I[WS(is, 4)];
+		    T9 = I[WS(is, 20)];
+		    Ta = T8 + T9;
+		    Tw = T8 - T9;
+		    Tb = I[WS(is, 28)];
+		    Tc = I[WS(is, 12)];
+		    Td = Tb + Tc;
+		    Tx = Tb - Tc;
+	       }
+	       Te = Ta + Td;
+	       T2o = Td - Ta;
+	       Ty = KP707106781 * (Tw + Tx);
+	       T1k = KP707106781 * (Tx - Tw);
+	  }
+	  {
+	       E Tp, TD, Ts, TE;
+	       {
+		    E Tn, To, Tq, Tr;
+		    Tn = I[WS(is, 30)];
+		    To = I[WS(is, 14)];
+		    Tp = Tn + To;
+		    TD = Tn - To;
+		    Tq = I[WS(is, 6)];
+		    Tr = I[WS(is, 22)];
+		    Ts = Tq + Tr;
+		    TE = Tq - Tr;
+	       }
+	       Tt = Tp + Ts;
+	       T2d = Tp - Ts;
+	       TF = FMA(KP923879532, TD, KP382683432 * TE);
+	       T1h = FNMS(KP923879532, TE, KP382683432 * TD);
+	  }
+	  {
+	       E Ti, TA, Tl, TB;
+	       {
+		    E Tg, Th, Tj, Tk;
+		    Tg = I[WS(is, 2)];
+		    Th = I[WS(is, 18)];
+		    Ti = Tg + Th;
+		    TA = Tg - Th;
+		    Tj = I[WS(is, 10)];
+		    Tk = I[WS(is, 26)];
+		    Tl = Tj + Tk;
+		    TB = Tj - Tk;
+	       }
+	       Tm = Ti + Tl;
+	       T2c = Ti - Tl;
+	       TC = FNMS(KP382683432, TB, KP923879532 * TA);
+	       T1i = FMA(KP382683432, TA, KP923879532 * TB);
+	  }
+	  {
+	       E T11, T1X, T1d, T1Y, T14, T20, T17, T21, T1a, T18;
+	       {
+		    E TZ, T10, T1b, T1c;
+		    TZ = I[WS(is, 31)];
+		    T10 = I[WS(is, 15)];
+		    T11 = TZ - T10;
+		    T1X = TZ + T10;
+		    T1b = I[WS(is, 7)];
+		    T1c = I[WS(is, 23)];
+		    T1d = T1b - T1c;
+		    T1Y = T1b + T1c;
+	       }
+	       {
+		    E T12, T13, T15, T16;
+		    T12 = I[WS(is, 3)];
+		    T13 = I[WS(is, 19)];
+		    T14 = T12 - T13;
+		    T20 = T12 + T13;
+		    T15 = I[WS(is, 27)];
+		    T16 = I[WS(is, 11)];
+		    T17 = T15 - T16;
+		    T21 = T15 + T16;
+	       }
+	       T1Z = T1X + T1Y;
+	       T22 = T20 + T21;
+	       T2k = T21 - T20;
+	       T2j = T1X - T1Y;
+	       T1a = KP707106781 * (T17 - T14);
+	       T1e = T1a - T1d;
+	       T1C = T1d + T1a;
+	       T18 = KP707106781 * (T14 + T17);
+	       T19 = T11 + T18;
+	       T1B = T11 - T18;
+	  }
+	  {
+	       E TK, T1Q, TW, T1R, TN, T1T, TQ, T1U, TT, TR;
+	       {
+		    E TI, TJ, TU, TV;
+		    TI = I[WS(is, 1)];
+		    TJ = I[WS(is, 17)];
+		    TK = TI - TJ;
+		    T1Q = TI + TJ;
+		    TU = I[WS(is, 9)];
+		    TV = I[WS(is, 25)];
+		    TW = TU - TV;
+		    T1R = TU + TV;
+	       }
+	       {
+		    E TL, TM, TO, TP;
+		    TL = I[WS(is, 5)];
+		    TM = I[WS(is, 21)];
+		    TN = TL - TM;
+		    T1T = TL + TM;
+		    TO = I[WS(is, 29)];
+		    TP = I[WS(is, 13)];
+		    TQ = TO - TP;
+		    T1U = TO + TP;
+	       }
+	       T1S = T1Q + T1R;
+	       T1V = T1T + T1U;
+	       T2h = T1U - T1T;
+	       T2g = T1Q - T1R;
+	       TT = KP707106781 * (TQ - TN);
+	       TX = TT - TW;
+	       T1z = TW + TT;
+	       TR = KP707106781 * (TN + TQ);
+	       TS = TK + TR;
+	       T1y = TK - TR;
+	  }
+	  {
+	       E Tf, Tu, T27, T28, T29, T2a;
+	       Tf = T7 + Te;
+	       Tu = Tm + Tt;
+	       T27 = Tf + Tu;
+	       T28 = T1S + T1V;
+	       T29 = T1Z + T22;
+	       T2a = T28 + T29;
+	       ro[WS(ros, 8)] = Tf - Tu;
+	       io[WS(ios, 8)] = T29 - T28;
+	       ro[WS(ros, 16)] = T27 - T2a;
+	       ro[0] = T27 + T2a;
+	  }
+	  {
+	       E T1P, T25, T24, T26, T1W, T23;
+	       T1P = T7 - Te;
+	       T25 = Tt - Tm;
+	       T1W = T1S - T1V;
+	       T23 = T1Z - T22;
+	       T24 = KP707106781 * (T1W + T23);
+	       T26 = KP707106781 * (T23 - T1W);
+	       ro[WS(ros, 12)] = T1P - T24;
+	       io[WS(ios, 12)] = T26 - T25;
+	       ro[WS(ros, 4)] = T1P + T24;
+	       io[WS(ios, 4)] = T25 + T26;
+	  }
+	  {
+	       E T2f, T2v, T2p, T2r, T2m, T2q, T2u, T2w, T2e, T2n;
+	       T2e = KP707106781 * (T2c + T2d);
+	       T2f = T2b + T2e;
+	       T2v = T2b - T2e;
+	       T2n = KP707106781 * (T2d - T2c);
+	       T2p = T2n - T2o;
+	       T2r = T2o + T2n;
+	       {
+		    E T2i, T2l, T2s, T2t;
+		    T2i = FMA(KP923879532, T2g, KP382683432 * T2h);
+		    T2l = FNMS(KP382683432, T2k, KP923879532 * T2j);
+		    T2m = T2i + T2l;
+		    T2q = T2l - T2i;
+		    T2s = FNMS(KP382683432, T2g, KP923879532 * T2h);
+		    T2t = FMA(KP382683432, T2j, KP923879532 * T2k);
+		    T2u = T2s + T2t;
+		    T2w = T2t - T2s;
+	       }
+	       ro[WS(ros, 14)] = T2f - T2m;
+	       io[WS(ios, 14)] = T2u - T2r;
+	       ro[WS(ros, 2)] = T2f + T2m;
+	       io[WS(ios, 2)] = T2r + T2u;
+	       io[WS(ios, 6)] = T2p + T2q;
+	       ro[WS(ros, 6)] = T2v + T2w;
+	       io[WS(ios, 10)] = T2q - T2p;
+	       ro[WS(ros, 10)] = T2v - T2w;
+	  }
+	  {
+	       E TH, T1t, T1s, T1u, T1g, T1o, T1n, T1p;
+	       {
+		    E Tz, TG, T1q, T1r;
+		    Tz = Tv + Ty;
+		    TG = TC + TF;
+		    TH = Tz + TG;
+		    T1t = Tz - TG;
+		    T1q = FNMS(KP195090322, TS, KP980785280 * TX);
+		    T1r = FMA(KP195090322, T19, KP980785280 * T1e);
+		    T1s = T1q + T1r;
+		    T1u = T1r - T1q;
+	       }
+	       {
+		    E TY, T1f, T1j, T1m;
+		    TY = FMA(KP980785280, TS, KP195090322 * TX);
+		    T1f = FNMS(KP195090322, T1e, KP980785280 * T19);
+		    T1g = TY + T1f;
+		    T1o = T1f - TY;
+		    T1j = T1h - T1i;
+		    T1m = T1k - T1l;
+		    T1n = T1j - T1m;
+		    T1p = T1m + T1j;
+	       }
+	       ro[WS(ros, 15)] = TH - T1g;
+	       io[WS(ios, 15)] = T1s - T1p;
+	       ro[WS(ros, 1)] = TH + T1g;
+	       io[WS(ios, 1)] = T1p + T1s;
+	       io[WS(ios, 7)] = T1n + T1o;
+	       ro[WS(ros, 7)] = T1t + T1u;
+	       io[WS(ios, 9)] = T1o - T1n;
+	       ro[WS(ros, 9)] = T1t - T1u;
+	  }
+	  {
+	       E T1x, T1N, T1M, T1O, T1E, T1I, T1H, T1J;
+	       {
+		    E T1v, T1w, T1K, T1L;
+		    T1v = Tv - Ty;
+		    T1w = T1i + T1h;
+		    T1x = T1v + T1w;
+		    T1N = T1v - T1w;
+		    T1K = FNMS(KP555570233, T1y, KP831469612 * T1z);
+		    T1L = FMA(KP555570233, T1B, KP831469612 * T1C);
+		    T1M = T1K + T1L;
+		    T1O = T1L - T1K;
+	       }
+	       {
+		    E T1A, T1D, T1F, T1G;
+		    T1A = FMA(KP831469612, T1y, KP555570233 * T1z);
+		    T1D = FNMS(KP555570233, T1C, KP831469612 * T1B);
+		    T1E = T1A + T1D;
+		    T1I = T1D - T1A;
+		    T1F = TF - TC;
+		    T1G = T1l + T1k;
+		    T1H = T1F - T1G;
+		    T1J = T1G + T1F;
+	       }
+	       ro[WS(ros, 13)] = T1x - T1E;
+	       io[WS(ios, 13)] = T1M - T1J;
+	       ro[WS(ros, 3)] = T1x + T1E;
+	       io[WS(ios, 3)] = T1J + T1M;
+	       io[WS(ios, 5)] = T1H + T1I;
+	       ro[WS(ros, 5)] = T1N + T1O;
+	       io[WS(ios, 11)] = T1I - T1H;
+	       ro[WS(ros, 11)] = T1N - T1O;
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 32, "r2hc_32", {140, 26, 16, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_32) (planner *p) {
+     X(kr2hc_register) (p, r2hc_32, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_4.c b/src/fftw3/rdft/codelets/r2hc/r2hc_4.c
new file mode 100644
index 0000000..0ef1567
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_4.c
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:40 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 4 -name r2hc_4 -include r2hc.h */
+
+/*
+ * This function contains 6 FP additions, 0 FP multiplications,
+ * (or, 6 additions, 0 multiplications, 0 fused multiply/add),
+ * 7 stack variables, and 8 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_4.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_4(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T2, T3, T4, T5, T6;
+	  T1 = I[0];
+	  T2 = I[WS(is, 2)];
+	  T3 = T1 + T2;
+	  T4 = I[WS(is, 1)];
+	  T5 = I[WS(is, 3)];
+	  T6 = T4 + T5;
+	  ro[WS(ros, 1)] = T1 - T2;
+	  io[WS(ios, 1)] = T5 - T4;
+	  ro[WS(ros, 2)] = T3 - T6;
+	  ro[0] = T3 + T6;
+     }
+}
+
+static const kr2hc_desc desc = { 4, "r2hc_4", {6, 0, 0, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_4) (planner *p) {
+     X(kr2hc_register) (p, r2hc_4, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_5.c b/src/fftw3/rdft/codelets/r2hc/r2hc_5.c
new file mode 100644
index 0000000..14f4849
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_5.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:40 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 5 -name r2hc_5 -include r2hc.h */
+
+/*
+ * This function contains 12 FP additions, 6 FP multiplications,
+ * (or, 9 additions, 3 multiplications, 3 fused multiply/add),
+ * 17 stack variables, and 10 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_5.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_5.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_5.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_5(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP250000000, +0.250000000000000000000000000000000000000000000);
+     DK(KP559016994, +0.559016994374947424102293417182819058860154590);
+     DK(KP587785252, +0.587785252292473129168705954639072768597652438);
+     DK(KP951056516, +0.951056516295153572116439333379382143405698634);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E Ta, T7, T8, T3, Tb, T6, T9, Tc;
+	  Ta = I[0];
+	  {
+	       E T1, T2, T4, T5;
+	       T1 = I[WS(is, 4)];
+	       T2 = I[WS(is, 1)];
+	       T7 = T2 + T1;
+	       T4 = I[WS(is, 2)];
+	       T5 = I[WS(is, 3)];
+	       T8 = T4 + T5;
+	       T3 = T1 - T2;
+	       Tb = T7 + T8;
+	       T6 = T4 - T5;
+	  }
+	  io[WS(ios, 1)] = FNMS(KP587785252, T6, KP951056516 * T3);
+	  ro[0] = Ta + Tb;
+	  io[WS(ios, 2)] = FMA(KP587785252, T3, KP951056516 * T6);
+	  T9 = KP559016994 * (T7 - T8);
+	  Tc = FNMS(KP250000000, Tb, Ta);
+	  ro[WS(ros, 1)] = T9 + Tc;
+	  ro[WS(ros, 2)] = Tc - T9;
+     }
+}
+
+static const kr2hc_desc desc = { 5, "r2hc_5", {9, 3, 3, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_5) (planner *p) {
+     X(kr2hc_register) (p, r2hc_5, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_6.c b/src/fftw3/rdft/codelets/r2hc/r2hc_6.c
new file mode 100644
index 0000000..5fadce3
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_6.c
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:40 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 6 -name r2hc_6 -include r2hc.h */
+
+/*
+ * This function contains 14 FP additions, 4 FP multiplications,
+ * (or, 12 additions, 2 multiplications, 2 fused multiply/add),
+ * 17 stack variables, and 12 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_6.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_6.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_6.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_6(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T3, Td, T9, Tc, T6, Tb, T1, T2, Ta, Te;
+	  T1 = I[0];
+	  T2 = I[WS(is, 3)];
+	  T3 = T1 - T2;
+	  Td = T1 + T2;
+	  {
+	       E T7, T8, T4, T5;
+	       T7 = I[WS(is, 4)];
+	       T8 = I[WS(is, 1)];
+	       T9 = T7 - T8;
+	       Tc = T7 + T8;
+	       T4 = I[WS(is, 2)];
+	       T5 = I[WS(is, 5)];
+	       T6 = T4 - T5;
+	       Tb = T4 + T5;
+	  }
+	  io[WS(ios, 1)] = KP866025403 * (T9 - T6);
+	  Ta = T6 + T9;
+	  ro[WS(ros, 1)] = FNMS(KP500000000, Ta, T3);
+	  ro[WS(ros, 3)] = T3 + Ta;
+	  io[WS(ios, 2)] = KP866025403 * (Tb - Tc);
+	  Te = Tb + Tc;
+	  ro[WS(ros, 2)] = FNMS(KP500000000, Te, Td);
+	  ro[0] = Td + Te;
+     }
+}
+
+static const kr2hc_desc desc = { 6, "r2hc_6", {12, 2, 2, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_6) (planner *p) {
+     X(kr2hc_register) (p, r2hc_6, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_7.c b/src/fftw3/rdft/codelets/r2hc/r2hc_7.c
new file mode 100644
index 0000000..8548e93
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_7.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:41 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 7 -name r2hc_7 -include r2hc.h */
+
+/*
+ * This function contains 24 FP additions, 18 FP multiplications,
+ * (or, 12 additions, 6 multiplications, 12 fused multiply/add),
+ * 20 stack variables, and 14 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_7.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_7.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_7.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_7(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP222520933, +0.222520933956314404288902564496794759466355569);
+     DK(KP900968867, +0.900968867902419126236102319507445051165919162);
+     DK(KP623489801, +0.623489801858733530525004884004239810632274731);
+     DK(KP433883739, +0.433883739117558120475768332848358754609990728);
+     DK(KP781831482, +0.781831482468029808708444526674057750232334519);
+     DK(KP974927912, +0.974927912181823607018131682993931217232785801);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, Ta, Tb, T4, Td, T7, Tc, T8, T9;
+	  T1 = I[0];
+	  T8 = I[WS(is, 1)];
+	  T9 = I[WS(is, 6)];
+	  Ta = T8 + T9;
+	  Tb = T9 - T8;
+	  {
+	       E T2, T3, T5, T6;
+	       T2 = I[WS(is, 2)];
+	       T3 = I[WS(is, 5)];
+	       T4 = T2 + T3;
+	       Td = T3 - T2;
+	       T5 = I[WS(is, 3)];
+	       T6 = I[WS(is, 4)];
+	       T7 = T5 + T6;
+	       Tc = T6 - T5;
+	  }
+	  io[WS(ios, 2)] = FNMS(KP781831482, Tc, KP974927912 * Tb) - (KP433883739 * Td);
+	  io[WS(ios, 1)] = FMA(KP781831482, Tb, KP974927912 * Td) + (KP433883739 * Tc);
+	  ro[WS(ros, 2)] = FMA(KP623489801, T7, T1) + FNMA(KP900968867, T4, KP222520933 * Ta);
+	  io[WS(ios, 3)] = FMA(KP433883739, Tb, KP974927912 * Tc) - (KP781831482 * Td);
+	  ro[WS(ros, 3)] = FMA(KP623489801, T4, T1) + FNMA(KP222520933, T7, KP900968867 * Ta);
+	  ro[WS(ros, 1)] = FMA(KP623489801, Ta, T1) + FNMA(KP900968867, T7, KP222520933 * T4);
+	  ro[0] = T1 + Ta + T4 + T7;
+     }
+}
+
+static const kr2hc_desc desc = { 7, "r2hc_7", {12, 6, 12, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_7) (planner *p) {
+     X(kr2hc_register) (p, r2hc_7, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_8.c b/src/fftw3/rdft/codelets/r2hc/r2hc_8.c
new file mode 100644
index 0000000..b502b94
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_8.c
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:41 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 8 -name r2hc_8 -include r2hc.h */
+
+/*
+ * This function contains 20 FP additions, 2 FP multiplications,
+ * (or, 20 additions, 2 multiplications, 0 fused multiply/add),
+ * 14 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_8.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_8(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T3, T7, Td, Tj, T6, Tg, Ta, Ti;
+	  {
+	       E T1, T2, Tb, Tc;
+	       T1 = I[0];
+	       T2 = I[WS(is, 4)];
+	       T3 = T1 + T2;
+	       T7 = T1 - T2;
+	       Tb = I[WS(is, 7)];
+	       Tc = I[WS(is, 3)];
+	       Td = Tb - Tc;
+	       Tj = Tb + Tc;
+	  }
+	  {
+	       E T4, T5, T8, T9;
+	       T4 = I[WS(is, 2)];
+	       T5 = I[WS(is, 6)];
+	       T6 = T4 + T5;
+	       Tg = T4 - T5;
+	       T8 = I[WS(is, 1)];
+	       T9 = I[WS(is, 5)];
+	       Ta = T8 - T9;
+	       Ti = T8 + T9;
+	  }
+	  ro[WS(ros, 2)] = T3 - T6;
+	  io[WS(ios, 2)] = Tj - Ti;
+	  {
+	       E Te, Tf, Th, Tk;
+	       Te = KP707106781 * (Ta + Td);
+	       ro[WS(ros, 3)] = T7 - Te;
+	       ro[WS(ros, 1)] = T7 + Te;
+	       Tf = KP707106781 * (Td - Ta);
+	       io[WS(ios, 1)] = Tf - Tg;
+	       io[WS(ios, 3)] = Tg + Tf;
+	       Th = T3 + T6;
+	       Tk = Ti + Tj;
+	       ro[WS(ros, 4)] = Th - Tk;
+	       ro[0] = Th + Tk;
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 8, "r2hc_8", {20, 2, 0, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_8) (planner *p) {
+     X(kr2hc_register) (p, r2hc_8, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/r2hc_9.c b/src/fftw3/rdft/codelets/r2hc/r2hc_9.c
new file mode 100644
index 0000000..df3808f
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/r2hc_9.c
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 21:56:40 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2hc -compact -variables 4 -n 9 -name r2hc_9 -include r2hc.h */
+
+/*
+ * This function contains 38 FP additions, 26 FP multiplications,
+ * (or, 21 additions, 9 multiplications, 17 fused multiply/add),
+ * 36 stack variables, and 18 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: r2hc_9.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_9.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ * $Id: r2hc_9.c,v 1.1 2008/10/17 06:12:34 scuri Exp $
+ */
+
+#include "r2hc.h"
+
+static void r2hc_9(const R *I, R *ro, R *io, stride is, stride ros, stride ios, int v, int ivs, int ovs)
+{
+     DK(KP939692620, +0.939692620785908384054109277324731469936208134);
+     DK(KP296198132, +0.296198132726023843175338011893050938967728390);
+     DK(KP342020143, +0.342020143325668733044099614682259580763083368);
+     DK(KP813797681, +0.813797681349373692844693217248393223289101568);
+     DK(KP984807753, +0.984807753012208059366743024589523013670643252);
+     DK(KP150383733, +0.150383733180435296639271897612501926072238258);
+     DK(KP642787609, +0.642787609686539326322643409907263432907559884);
+     DK(KP663413948, +0.663413948168938396205421319635891297216863310);
+     DK(KP852868531, +0.852868531952443209628250963940074071936020296);
+     DK(KP173648177, +0.173648177666930348851716626769314796000375677);
+     DK(KP556670399, +0.556670399226419366452912952047023132968291906);
+     DK(KP766044443, +0.766044443118978035202392650555416673935832457);
+     DK(KP866025403, +0.866025403784438646763723170752936183471402627);
+     DK(KP500000000, +0.500000000000000000000000000000000000000000000);
+     int i;
+     for (i = v; i > 0; i = i - 1, I = I + ivs, ro = ro + ovs, io = io + ovs) {
+	  E T1, T4, Tr, Ta, Tl, Ti, Tf, Tk, Tj, T2, T3, T5, Tg;
+	  T1 = I[0];
+	  T2 = I[WS(is, 3)];
+	  T3 = I[WS(is, 6)];
+	  T4 = T2 + T3;
+	  Tr = T3 - T2;
+	  {
+	       E T6, T7, T8, T9;
+	       T6 = I[WS(is, 1)];
+	       T7 = I[WS(is, 4)];
+	       T8 = I[WS(is, 7)];
+	       T9 = T7 + T8;
+	       Ta = T6 + T9;
+	       Tl = T8 - T7;
+	       Ti = FNMS(KP500000000, T9, T6);
+	  }
+	  {
+	       E Tb, Tc, Td, Te;
+	       Tb = I[WS(is, 2)];
+	       Tc = I[WS(is, 5)];
+	       Td = I[WS(is, 8)];
+	       Te = Tc + Td;
+	       Tf = Tb + Te;
+	       Tk = FNMS(KP500000000, Te, Tb);
+	       Tj = Td - Tc;
+	  }
+	  io[WS(ios, 3)] = KP866025403 * (Tf - Ta);
+	  T5 = T1 + T4;
+	  Tg = Ta + Tf;
+	  ro[WS(ros, 3)] = FNMS(KP500000000, Tg, T5);
+	  ro[0] = T5 + Tg;
+	  {
+	       E Tt, Th, Tm, Tn, To, Tp, Tq, Ts;
+	       Tt = KP866025403 * Tr;
+	       Th = FNMS(KP500000000, T4, T1);
+	       Tm = FMA(KP766044443, Ti, KP556670399 * Tl);
+	       Tn = FMA(KP173648177, Tk, KP852868531 * Tj);
+	       To = Tm + Tn;
+	       Tp = FNMS(KP642787609, Ti, KP663413948 * Tl);
+	       Tq = FNMS(KP984807753, Tk, KP150383733 * Tj);
+	       Ts = Tp + Tq;
+	       ro[WS(ros, 1)] = Th + To;
+	       io[WS(ios, 1)] = Tt + Ts;
+	       ro[WS(ros, 4)] = FMA(KP866025403, Tp - Tq, Th) - (KP500000000 * To);
+	       io[WS(ios, 4)] = FNMS(KP500000000, Ts, KP866025403 * (Tr + (Tn - Tm)));
+	       io[WS(ios, 2)] = FNMS(KP342020143, Tk, KP813797681 * Tj) + FNMA(KP150383733, Tl, KP984807753 * Ti) - Tt;
+	       ro[WS(ros, 2)] = FMA(KP173648177, Ti, Th) + FNMA(KP296198132, Tj, KP939692620 * Tk) - (KP852868531 * Tl);
+	  }
+     }
+}
+
+static const kr2hc_desc desc = { 9, "r2hc_9", {21, 9, 17, 0}, &GENUS, 0, 0, 0, 0, 0 };
+
+void X(codelet_r2hc_9) (planner *p) {
+     X(kr2hc_register) (p, r2hc_9, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2hc/rhcodlist.c b/src/fftw3/rdft/codelets/r2hc/rhcodlist.c
new file mode 100644
index 0000000..a4ac1bc
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hc/rhcodlist.c
@@ -0,0 +1,114 @@
+#include "ifftw.h"
+
+extern void X(codelet_r2hc_2)(planner *);
+extern void X(codelet_r2hc_3)(planner *);
+extern void X(codelet_r2hc_4)(planner *);
+extern void X(codelet_r2hc_5)(planner *);
+extern void X(codelet_r2hc_6)(planner *);
+extern void X(codelet_r2hc_7)(planner *);
+extern void X(codelet_r2hc_8)(planner *);
+extern void X(codelet_r2hc_9)(planner *);
+extern void X(codelet_r2hc_10)(planner *);
+extern void X(codelet_r2hc_11)(planner *);
+extern void X(codelet_r2hc_12)(planner *);
+extern void X(codelet_r2hc_13)(planner *);
+extern void X(codelet_r2hc_14)(planner *);
+extern void X(codelet_r2hc_15)(planner *);
+extern void X(codelet_r2hc_16)(planner *);
+extern void X(codelet_r2hc_32)(planner *);
+extern void X(codelet_mr2hc_32)(planner *);
+extern void X(codelet_mr2hc_64)(planner *);
+extern void X(codelet_mr2hc_128)(planner *);
+extern void X(codelet_hf_2)(planner *);
+extern void X(codelet_hf_3)(planner *);
+extern void X(codelet_hf_4)(planner *);
+extern void X(codelet_hf_5)(planner *);
+extern void X(codelet_hf_6)(planner *);
+extern void X(codelet_hf_7)(planner *);
+extern void X(codelet_hf_8)(planner *);
+extern void X(codelet_hf_9)(planner *);
+extern void X(codelet_hf_10)(planner *);
+extern void X(codelet_hf_12)(planner *);
+extern void X(codelet_hf_15)(planner *);
+extern void X(codelet_hf_16)(planner *);
+extern void X(codelet_hf_32)(planner *);
+extern void X(codelet_hf_64)(planner *);
+extern void X(codelet_hf2_4)(planner *);
+extern void X(codelet_hf2_8)(planner *);
+extern void X(codelet_hf2_16)(planner *);
+extern void X(codelet_hf2_32)(planner *);
+extern void X(codelet_hf2_64)(planner *);
+extern void X(codelet_r2hcII_2)(planner *);
+extern void X(codelet_r2hcII_3)(planner *);
+extern void X(codelet_r2hcII_4)(planner *);
+extern void X(codelet_r2hcII_5)(planner *);
+extern void X(codelet_r2hcII_6)(planner *);
+extern void X(codelet_r2hcII_7)(planner *);
+extern void X(codelet_r2hcII_8)(planner *);
+extern void X(codelet_r2hcII_9)(planner *);
+extern void X(codelet_r2hcII_10)(planner *);
+extern void X(codelet_r2hcII_12)(planner *);
+extern void X(codelet_r2hcII_15)(planner *);
+extern void X(codelet_r2hcII_16)(planner *);
+extern void X(codelet_r2hcII_32)(planner *);
+extern void X(codelet_mr2hcII_32)(planner *);
+extern void X(codelet_mr2hcII_64)(planner *);
+
+
+extern const solvtab X(solvtab_rdft_r2hc);
+const solvtab X(solvtab_rdft_r2hc) = {
+   SOLVTAB(X(codelet_r2hc_2)),
+   SOLVTAB(X(codelet_r2hc_3)),
+   SOLVTAB(X(codelet_r2hc_4)),
+   SOLVTAB(X(codelet_r2hc_5)),
+   SOLVTAB(X(codelet_r2hc_6)),
+   SOLVTAB(X(codelet_r2hc_7)),
+   SOLVTAB(X(codelet_r2hc_8)),
+   SOLVTAB(X(codelet_r2hc_9)),
+   SOLVTAB(X(codelet_r2hc_10)),
+   SOLVTAB(X(codelet_r2hc_11)),
+   SOLVTAB(X(codelet_r2hc_12)),
+   SOLVTAB(X(codelet_r2hc_13)),
+   SOLVTAB(X(codelet_r2hc_14)),
+   SOLVTAB(X(codelet_r2hc_15)),
+   SOLVTAB(X(codelet_r2hc_16)),
+   SOLVTAB(X(codelet_r2hc_32)),
+   SOLVTAB(X(codelet_mr2hc_32)),
+   SOLVTAB(X(codelet_mr2hc_64)),
+   SOLVTAB(X(codelet_mr2hc_128)),
+   SOLVTAB(X(codelet_hf_2)),
+   SOLVTAB(X(codelet_hf_3)),
+   SOLVTAB(X(codelet_hf_4)),
+   SOLVTAB(X(codelet_hf_5)),
+   SOLVTAB(X(codelet_hf_6)),
+   SOLVTAB(X(codelet_hf_7)),
+   SOLVTAB(X(codelet_hf_8)),
+   SOLVTAB(X(codelet_hf_9)),
+   SOLVTAB(X(codelet_hf_10)),
+   SOLVTAB(X(codelet_hf_12)),
+   SOLVTAB(X(codelet_hf_15)),
+   SOLVTAB(X(codelet_hf_16)),
+   SOLVTAB(X(codelet_hf_32)),
+   SOLVTAB(X(codelet_hf_64)),
+   SOLVTAB(X(codelet_hf2_4)),
+   SOLVTAB(X(codelet_hf2_8)),
+   SOLVTAB(X(codelet_hf2_16)),
+   SOLVTAB(X(codelet_hf2_32)),
+   SOLVTAB(X(codelet_hf2_64)),
+   SOLVTAB(X(codelet_r2hcII_2)),
+   SOLVTAB(X(codelet_r2hcII_3)),
+   SOLVTAB(X(codelet_r2hcII_4)),
+   SOLVTAB(X(codelet_r2hcII_5)),
+   SOLVTAB(X(codelet_r2hcII_6)),
+   SOLVTAB(X(codelet_r2hcII_7)),
+   SOLVTAB(X(codelet_r2hcII_8)),
+   SOLVTAB(X(codelet_r2hcII_9)),
+   SOLVTAB(X(codelet_r2hcII_10)),
+   SOLVTAB(X(codelet_r2hcII_12)),
+   SOLVTAB(X(codelet_r2hcII_15)),
+   SOLVTAB(X(codelet_r2hcII_16)),
+   SOLVTAB(X(codelet_r2hcII_32)),
+   SOLVTAB(X(codelet_mr2hcII_32)),
+   SOLVTAB(X(codelet_mr2hcII_64)),
+   SOLVTAB_END
+};
diff --git a/src/fftw3/rdft/codelets/r2hcII.h b/src/fftw3/rdft/codelets/r2hcII.h
new file mode 100644
index 0000000..cb90935
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2hcII.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+
+#define GENUS X(rdft_r2hcII_genus)
+extern const kr2hcII_genus GENUS;
diff --git a/src/fftw3/rdft/codelets/r2r.c b/src/fftw3/rdft/codelets/r2r.c
new file mode 100644
index 0000000..18d9528
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2r.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "codelet-rdft.h"
+#include "r2r.h"
+
+static int okp(const kr2r_desc *d,
+	       const R *I,
+	       const R *O,
+	       int is, int os, int vl, int ivs, int ovs)
+{
+     UNUSED(I); UNUSED(O); UNUSED(vl);
+     return (1
+	     && (!d->is || (d->is == is))
+	     && (!d->os || (d->os == os))
+	     && (!d->ivs || (d->ivs == ivs))
+	     && (!d->ovs || (d->ovs == ovs))
+	  );
+}
+
+const kr2r_genus GENUS = { okp, 1 };
diff --git a/src/fftw3/rdft/codelets/r2r.h b/src/fftw3/rdft/codelets/r2r.h
new file mode 100644
index 0000000..614427c
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2r.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+
+#define GENUS X(rdft_r2r_genus)
+extern const kr2r_genus GENUS;
diff --git a/src/fftw3/rdft/codelets/r2r/e01_8.c b/src/fftw3/rdft/codelets/r2r/e01_8.c
new file mode 100644
index 0000000..b48371c
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2r/e01_8.c
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:22:02 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2r -compact -variables 4 -redft01 -n 8 -name e01_8 -include r2r.h */
+
+/*
+ * This function contains 26 FP additions, 15 FP multiplications,
+ * (or, 20 additions, 9 multiplications, 6 fused multiply/add),
+ * 27 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: e01_8.c,v 1.1 2008/10/17 06:13:17 scuri Exp $
+ * $Id: e01_8.c,v 1.1 2008/10/17 06:13:17 scuri Exp $
+ * $Id: e01_8.c,v 1.1 2008/10/17 06:13:17 scuri Exp $
+ */
+
+#include "r2r.h"
+
+static void e01_8_0(const R *I, R *O, stride istride, stride ostride)
+{
+     DK(KP1_662939224, +1.662939224605090474157576755235811513477121624);
+     DK(KP1_111140466, +1.111140466039204449485661627897065748749874382);
+     DK(KP390180644, +0.390180644032256535696569736954044481855383236);
+     DK(KP1_961570560, +1.961570560806460898252364472268478073947867462);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     {
+	  E T7, Tl, T4, Tk, Td, To, Tg, Tn;
+	  {
+	       E T5, T6, T1, T3, T2;
+	       T5 = I[WS(istride, 2)];
+	       T6 = I[WS(istride, 6)];
+	       T7 = FMA(KP1_847759065, T5, KP765366864 * T6);
+	       Tl = FNMS(KP1_847759065, T6, KP765366864 * T5);
+	       T1 = I[0];
+	       T2 = I[WS(istride, 4)];
+	       T3 = KP1_414213562 * T2;
+	       T4 = T1 + T3;
+	       Tk = T1 - T3;
+	       {
+		    E T9, Tf, Tc, Te, Ta, Tb;
+		    T9 = I[WS(istride, 1)];
+		    Tf = I[WS(istride, 7)];
+		    Ta = I[WS(istride, 5)];
+		    Tb = I[WS(istride, 3)];
+		    Tc = KP707106781 * (Ta + Tb);
+		    Te = KP707106781 * (Ta - Tb);
+		    Td = T9 + Tc;
+		    To = Te + Tf;
+		    Tg = Te - Tf;
+		    Tn = T9 - Tc;
+	       }
+	  }
+	  {
+	       E T8, Th, Tq, Tr;
+	       T8 = T4 + T7;
+	       Th = FNMS(KP390180644, Tg, KP1_961570560 * Td);
+	       O[WS(ostride, 7)] = T8 - Th;
+	       O[0] = T8 + Th;
+	       Tq = Tk - Tl;
+	       Tr = FMA(KP1_111140466, Tn, KP1_662939224 * To);
+	       O[WS(ostride, 5)] = Tq - Tr;
+	       O[WS(ostride, 2)] = Tq + Tr;
+	  }
+	  {
+	       E Ti, Tj, Tm, Tp;
+	       Ti = T4 - T7;
+	       Tj = FMA(KP390180644, Td, KP1_961570560 * Tg);
+	       O[WS(ostride, 4)] = Ti - Tj;
+	       O[WS(ostride, 3)] = Ti + Tj;
+	       Tm = Tk + Tl;
+	       Tp = FNMS(KP1_111140466, To, KP1_662939224 * Tn);
+	       O[WS(ostride, 6)] = Tm - Tp;
+	       O[WS(ostride, 1)] = Tm + Tp;
+	  }
+     }
+}
+
+static void e01_8(const R *I, R *O, stride is, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  e01_8_0(I, O, is, os);
+	  I += ivs;
+	  O += ovs;
+     }
+}
+
+static const kr2r_desc desc = { 8, "e01_8", {20, 9, 6, 0}, &GENUS, REDFT01, 0, 0, 0, 0 };
+
+void X(codelet_e01_8) (planner *p) {
+     X(kr2r_register) (p, e01_8, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2r/e10_8.c b/src/fftw3/rdft/codelets/r2r/e10_8.c
new file mode 100644
index 0000000..6d7be49
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2r/e10_8.c
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* This file was automatically generated --- DO NOT EDIT */
+/* Generated on Sat Jul  5 22:22:02 EDT 2003 */
+
+#include "codelet-rdft.h"
+
+/* Generated by: /homee/stevenj/cvs/fftw3.0.1/genfft/gen_r2r -compact -variables 4 -redft10 -n 8 -name e10_8 -include r2r.h */
+
+/*
+ * This function contains 26 FP additions, 16 FP multiplications,
+ * (or, 20 additions, 10 multiplications, 6 fused multiply/add),
+ * 27 stack variables, and 16 memory accesses
+ */
+/*
+ * Generator Id's : 
+ * $Id: e10_8.c,v 1.1 2008/10/17 06:13:17 scuri Exp $
+ * $Id: e10_8.c,v 1.1 2008/10/17 06:13:17 scuri Exp $
+ * $Id: e10_8.c,v 1.1 2008/10/17 06:13:17 scuri Exp $
+ */
+
+#include "r2r.h"
+
+static void e10_8_0(const R *I, R *O, stride istride, stride ostride)
+{
+     DK(KP765366864, +0.765366864730179543456919968060797733522689125);
+     DK(KP1_847759065, +1.847759065022573512256366378793576573644833252);
+     DK(KP390180644, +0.390180644032256535696569736954044481855383236);
+     DK(KP1_961570560, +1.961570560806460898252364472268478073947867462);
+     DK(KP2_000000000, +2.000000000000000000000000000000000000000000000);
+     DK(KP1_414213562, +1.414213562373095048801688724209698078569671875);
+     DK(KP1_111140466, +1.111140466039204449485661627897065748749874382);
+     DK(KP1_662939224, +1.662939224605090474157576755235811513477121624);
+     DK(KP707106781, +0.707106781186547524400844362104849039284835938);
+     {
+	  E T3, Tj, Tf, Tk, Ta, Tn, Tc, Tm;
+	  {
+	       E T1, T2, Td, Te;
+	       T1 = I[0];
+	       T2 = I[WS(istride, 7)];
+	       T3 = T1 - T2;
+	       Tj = T1 + T2;
+	       Td = I[WS(istride, 4)];
+	       Te = I[WS(istride, 3)];
+	       Tf = Td - Te;
+	       Tk = Td + Te;
+	       {
+		    E T4, T5, T6, T7, T8, T9;
+		    T4 = I[WS(istride, 2)];
+		    T5 = I[WS(istride, 5)];
+		    T6 = T4 - T5;
+		    T7 = I[WS(istride, 1)];
+		    T8 = I[WS(istride, 6)];
+		    T9 = T7 - T8;
+		    Ta = KP707106781 * (T6 + T9);
+		    Tn = T7 + T8;
+		    Tc = KP707106781 * (T6 - T9);
+		    Tm = T4 + T5;
+	       }
+	  }
+	  {
+	       E Tb, Tg, Tp, Tq;
+	       Tb = T3 - Ta;
+	       Tg = Tc - Tf;
+	       O[WS(ostride, 3)] = FNMS(KP1_111140466, Tg, KP1_662939224 * Tb);
+	       O[WS(ostride, 5)] = FMA(KP1_662939224, Tg, KP1_111140466 * Tb);
+	       Tp = Tj + Tk;
+	       Tq = Tm + Tn;
+	       O[WS(ostride, 4)] = KP1_414213562 * (Tp - Tq);
+	       O[0] = KP2_000000000 * (Tp + Tq);
+	  }
+	  {
+	       E Th, Ti, Tl, To;
+	       Th = T3 + Ta;
+	       Ti = Tf + Tc;
+	       O[WS(ostride, 1)] = FNMS(KP390180644, Ti, KP1_961570560 * Th);
+	       O[WS(ostride, 7)] = FMA(KP1_961570560, Ti, KP390180644 * Th);
+	       Tl = Tj - Tk;
+	       To = Tm - Tn;
+	       O[WS(ostride, 2)] = FNMS(KP765366864, To, KP1_847759065 * Tl);
+	       O[WS(ostride, 6)] = FMA(KP765366864, Tl, KP1_847759065 * To);
+	  }
+     }
+}
+
+static void e10_8(const R *I, R *O, stride is, stride os, int v, int ivs, int ovs)
+{
+     int i;
+     for (i = v; i > 0; --i) {
+	  e10_8_0(I, O, is, os);
+	  I += ivs;
+	  O += ovs;
+     }
+}
+
+static const kr2r_desc desc = { 8, "e10_8", {20, 10, 6, 0}, &GENUS, REDFT10, 0, 0, 0, 0 };
+
+void X(codelet_e10_8) (planner *p) {
+     X(kr2r_register) (p, e10_8, &desc);
+}
diff --git a/src/fftw3/rdft/codelets/r2r/rrcodlist.c b/src/fftw3/rdft/codelets/r2r/rrcodlist.c
new file mode 100644
index 0000000..28c1ebc
--- /dev/null
+++ b/src/fftw3/rdft/codelets/r2r/rrcodlist.c
@@ -0,0 +1,12 @@
+#include "ifftw.h"
+
+extern void X(codelet_e01_8)(planner *);
+extern void X(codelet_e10_8)(planner *);
+
+
+extern const solvtab X(solvtab_rdft_r2r);
+const solvtab X(solvtab_rdft_r2r) = {
+   SOLVTAB(X(codelet_e01_8)),
+   SOLVTAB(X(codelet_e10_8)),
+   SOLVTAB_END
+};
diff --git a/src/fftw3/rdft/dft-r2hc.c b/src/fftw3/rdft/dft-r2hc.c
new file mode 100644
index 0000000..10abf68
--- /dev/null
+++ b/src/fftw3/rdft/dft-r2hc.c
@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: dft-r2hc.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* Compute the complex DFT by combining R2HC RDFTs on the real
+   and imaginary parts.   This could be useful for people just wanting
+   to link to the real codelets and not the complex ones.  It could
+   also even be faster than the complex algorithms for split (as opposed
+   to interleaved) real/imag complex data. */
+
+#include "rdft.h"
+#include "dft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_dft super;
+     plan *cld;
+     int os;
+     int n;
+} P;
+
+static void apply(const plan *ego_, R *ri, R *ii, R *ro, R *io)
+{
+     const P *ego = (const P *) ego_;
+     int os;
+     int i, n;
+
+     UNUSED(ii);
+
+     { /* transform vector of real & imag parts: */
+	  plan_rdft *cld = (plan_rdft *) ego->cld;
+	  cld->apply((plan *) cld, ri, ro);
+     }
+
+     os = ego->os;
+     n = ego->n;
+     for (i = 1; i < (n + 1)/2; ++i) {
+	  R rop, iop, iom, rom;
+	  rop = ro[os * i];
+	  iop = io[os * i];
+	  rom = ro[os * (n - i)];
+	  iom = io[os * (n - i)];
+	  ro[os * i] = rop - iom;
+	  io[os * i] = iop + rom;
+	  ro[os * (n - i)] = rop + iom;
+	  io[os * (n - i)] = iop - rom;
+     }
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cld, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(dft-r2hc-%d%(%p%))", ego->n, ego->cld);
+}
+
+#define ALLOW_RANK0 0 /* disable for now, subject to testing */
+
+static int applicable0(const problem *p_)
+{
+     if (DFTP(p_)) {
+          const problem_dft *p = (const problem_dft *) p_;
+          return ((p->sz->rnk == 1 && p->vecsz->rnk == 0)
+#if ALLOW_RANK0
+		  || p->sz->rnk == 0
+#endif
+	       );
+     }
+
+     return 0;
+}
+
+static int split(R *r, R *i, int n, int s)
+{
+     return ((r > i ? r - i : i - r) >= ((int)n) * (s > 0 ? s : -s));
+}
+
+static int applicable(const problem *p_, const planner *plnr)
+{
+     if (!applicable0(p_)) return 0;
+
+     {
+	  const problem_dft *p = (const problem_dft *) p_;
+	  if (NO_UGLYP(plnr) && DFT_R2HC_ICKYP(plnr)) return 0;
+
+	  if (p->sz->rnk == 1 &&
+	      split(p->ri, p->ii, p->sz->dims[0].n, p->sz->dims[0].is) &&
+	      split(p->ro, p->io, p->sz->dims[0].n, p->sz->dims[0].os))
+	       return 1;
+
+	  return !(NO_UGLYP(plnr));
+     }
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     P *pln;
+     const problem_dft *p;
+     plan *cld;
+
+     static const plan_adt padt = {
+	  X(dft_solve), awake, print, destroy
+     };
+
+     UNUSED(ego_);
+     if (!applicable(p_, plnr))
+          return (plan *)0;
+
+     p = (const problem_dft *) p_;
+
+     {
+	  tensor *ri_vec = X(mktensor_1d)(2, p->ii - p->ri, p->io - p->ro);
+	  tensor *cld_vec = X(tensor_append)(ri_vec, p->vecsz);
+	  cld = X(mkplan_d)(plnr, 
+			    X(mkproblem_rdft_1)(p->sz, cld_vec, 
+						p->ri, p->ro, R2HC));
+	  X(tensor_destroy2)(ri_vec, cld_vec);
+     }
+     if (!cld) return (plan *)0;
+
+     pln = MKPLAN_DFT(P, &padt, apply);
+
+#if ALLOW_RANK0
+     if (p->sz->rnk == 0) {
+	  pln->n = 1;
+	  pln->os = 0;
+     }
+     else
+#endif
+     {
+	  pln->n = p->sz->dims[0].n;
+	  pln->os = p->sz->dims[0].os;
+     }
+
+     pln->cld = cld;
+     
+     pln->super.super.ops = cld->ops;
+     pln->super.super.ops.other += 8 * ((pln->n - 1)/2);
+     pln->super.super.ops.add += 4 * ((pln->n - 1)/2);
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(dft_r2hc_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/rdft/dht-r2hc.c b/src/fftw3/rdft/dht-r2hc.c
new file mode 100644
index 0000000..e66e614
--- /dev/null
+++ b/src/fftw3/rdft/dht-r2hc.c
@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: dht-r2hc.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* Solve a DHT problem (Discrete Hartley Transform) via post-processing
+   of an R2HC problem. */
+
+#include "rdft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cld;
+     int os;
+     int n;
+} P;
+
+static void apply(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int os = ego->os;
+     int i, n = ego->n;
+
+     {
+	  plan_rdft *cld = (plan_rdft *) ego->cld;
+	  cld->apply((plan *) cld, I, O);
+     }
+
+     for (i = 1; i < n - i; ++i) {
+	  E a, b;
+	  a = O[os * i];
+	  b = O[os * (n - i)];
+#if FFT_SIGN == -1
+	  O[os * i] = a - b;
+	  O[os * (n - i)] = a + b;
+#else
+	  O[os * i] = a + b;
+	  O[os * (n - i)] = a - b;
+#endif
+     }
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cld, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(dht-r2hc-%d%(%p%))", ego->n, ego->cld);
+}
+
+static int applicable0(const problem *p_, const planner *plnr)
+{
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && !NO_DHT_R2HCP(plnr)
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk == 0
+		  && p->kind[0] == DHT
+	       );
+     }
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p, const planner *plnr)
+{
+     UNUSED(ego);
+     return (!NO_UGLYP(plnr) && applicable0(p, plnr));
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     P *pln;
+     const problem_rdft *p;
+     plan *cld;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+          return (plan *)0;
+
+     p = (const problem_rdft *) p_;
+
+     /* stop infinite loops with rdft-dht.c */
+     plnr->problem_flags |= NO_DHT_R2HC; 
+
+     cld = X(mkplan_d)(plnr, 
+		       X(mkproblem_rdft_1)(p->sz, p->vecsz, p->I, p->O, R2HC));
+     if (!cld) return (plan *)0;
+
+     pln = MKPLAN_RDFT(P, &padt, apply);
+
+     pln->n = p->sz->dims[0].n;
+     pln->os = p->sz->dims[0].os;
+     pln->cld = cld;
+     
+     pln->super.super.ops = cld->ops;
+     pln->super.super.ops.other += 4 * ((pln->n - 1)/2);
+     pln->super.super.ops.add += 2 * ((pln->n - 1)/2);
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(dht_r2hc_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/rdft/dht-rader.c b/src/fftw3/rdft/dht-rader.c
new file mode 100644
index 0000000..b9a2a74
--- /dev/null
+++ b/src/fftw3/rdft/dht-rader.c
@@ -0,0 +1,344 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "rdft.h"
+
+/*
+ * Compute DHTs of prime sizes using Rader's trick: turn them
+ * into convolutions of size n - 1, which we then perform via a pair
+ * of FFTs.   (We can then do prime real FFTs via rdft-dht.c.)
+ */
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+
+     plan *cld1, *cld2;
+     R *omega;
+     int n, g, ginv;
+     int is, os;
+     plan *cld_omega;
+} P;
+
+static rader_tl *omegas = 0;
+
+/***************************************************************************/
+
+/* If R2HC_ONLY_CONV is 1, we use a trick to perform the convolution
+   purely in terms of R2HC transforms, as opposed to R2HC followed by H2RC.
+   This requires a few more operations, but allows us to share the same
+   plan/codelets for both Rader children. */
+#define R2HC_ONLY_CONV 1
+
+static void apply(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int r = ego->n;
+     int is = ego->is, os;
+     int k, gpower, g;
+     R *buf, *omega;
+     R r0;
+
+     buf = (R *) MALLOC(sizeof(R) * (r - 1), BUFFERS);
+
+     /* First, permute the input, storing in buf: */
+     g = ego->g; 
+     for (gpower = 1, k = 0; k < r - 1; ++k, gpower = MULMOD(gpower, g, r)) {
+	  buf[k] = I[gpower * is];
+     }
+     /* gpower == g^(r-1) mod r == 1 */;
+
+     os = ego->os;
+
+     /* compute RDFT of buf, storing in output (except DC): */
+     {
+	    plan_rdft *cld = (plan_rdft *) ego->cld1;
+	    cld->apply((plan *) cld, buf, O + os);
+     }
+
+     /* set output DC component: */
+     O[0] = (r0 = I[0]) + O[os];
+
+     /* now, multiply by omega: */
+     omega = ego->omega;
+
+     O[(0 + 1) * os] *= omega[0];
+#if R2HC_ONLY_CONV
+     for (k = 1; k < (r - 1)/2; ++k) {
+	  E rB, iB, rW, iW, a, b;
+	  rW = omega[k];
+	  iW = omega[(r-1) - k];
+	  rB = O[(k + 1) * os];
+	  iB = O[((r-1) - k + 1) * os];
+	  a = rW * rB - iW * iB;
+	  b = rW * iB + iW * rB;
+	  O[(k + 1) * os] = a + b;
+	  O[((r-1) - k + 1) * os] = a - b;
+     }
+#else
+     for (k = 1; k < (r - 1)/2; ++k) {
+	  E rB, iB, rW, iW;
+	  rW = omega[k];
+	  iW = omega[(r-1) - k];
+	  rB = O[(k + 1) * os];
+	  iB = O[((r-1) - k + 1) * os];
+	  O[(k + 1) * os] = rW * rB - iW * iB;
+	  O[((r-1) - k + 1) * os] = rW * iB + iW * rB;
+     }
+#endif
+     /* Nyquist component: */
+     O[(k + 1) * os] *= omega[k]; /* k == (r-1)/2, since r-1 is even */
+     
+     /* this will add input[0] to all of the outputs after the ifft */
+     O[os] += r0;
+
+     /* inverse FFT: */
+     {
+	    plan_rdft *cld = (plan_rdft *) ego->cld2;
+	    cld->apply((plan *) cld, O + os, buf);
+     }
+     
+     /* do inverse permutation to unshuffle the output: */
+     A(gpower == 1);
+#if R2HC_ONLY_CONV
+     O[os] = buf[0];
+     gpower = g = ego->ginv;
+     for (k = 1; k < (r - 1)/2; ++k, gpower = MULMOD(gpower, g, r)) {
+	  O[gpower * os] = buf[k] + buf[r - 1 - k];
+     }
+     O[gpower * os] = buf[k];
+     ++k, gpower = MULMOD(gpower, g, r);
+     for (; k < r - 1; ++k, gpower = MULMOD(gpower, g, r)) {
+	  O[gpower * os] = buf[r - 1 - k] - buf[k];
+     }
+#else
+     g = ego->ginv;
+     for (k = 0; k < r - 1; ++k, gpower = MULMOD(gpower, g, r)) {
+	  O[gpower * os] = buf[k];
+     }
+#endif
+     A(gpower == 1);
+
+     X(ifree)(buf);
+}
+
+static R *mkomega(plan *p_, int n, int ginv)
+{
+     plan_rdft *p = (plan_rdft *) p_;
+     R *omega;
+     int i, gpower;
+     trigreal scale;
+
+     if ((omega = X(rader_tl_find)(n, n, ginv, omegas))) 
+	  return omega;
+
+     omega = (R *)MALLOC(sizeof(R) * (n - 1), TWIDDLES);
+
+     scale = n - 1.0; /* normalization for convolution */
+
+     for (i = 0, gpower = 1; i < n-1; ++i, gpower = MULMOD(gpower, ginv, n)) {
+	  omega[i] = (X(cos2pi)(gpower, n) + X(sin2pi)(gpower, n)) / scale;
+     }
+     A(gpower == 1);
+
+     AWAKE(p_, 1);
+     p->apply(p_, omega, omega);
+     AWAKE(p_, 0);
+
+     X(rader_tl_insert)(n, n, ginv, omega, &omegas);
+     return omega;
+}
+
+static void free_omega(R *omega)
+{
+     X(rader_tl_delete)(omega, &omegas);
+}
+
+/***************************************************************************/
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+
+     AWAKE(ego->cld1, flg);
+     AWAKE(ego->cld2, flg);
+
+     if (flg) {
+	  if (!ego->omega) 
+	       ego->omega = mkomega(ego->cld_omega,ego->n,ego->ginv);
+     } else {
+	  free_omega(ego->omega);
+	  ego->omega = 0;
+     }
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld_omega);
+     X(plan_destroy_internal)(ego->cld2);
+     X(plan_destroy_internal)(ego->cld1);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+
+     p->print(p, "(dht-rader-%d%ois=%oos=%(%p%)",
+              ego->n, ego->is, ego->os, ego->cld1);
+     if (ego->cld2 != ego->cld1)
+          p->print(p, "%(%p%)", ego->cld2);
+     if (ego->cld_omega != ego->cld1 && ego->cld_omega != ego->cld2)
+          p->print(p, "%(%p%)", ego->cld_omega);
+     p->putchr(p, ')');
+}
+
+static int applicable0(const problem *p_)
+{
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk == 0
+		  && p->kind[0] == DHT
+		  && X(is_prime)(p->sz->dims[0].n)
+		  && p->sz->dims[0].n > 2
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p, const planner *plnr)
+{
+     UNUSED(ego);
+     return (!NO_UGLYP(plnr) && applicable0(p));
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const problem_rdft *p = (const problem_rdft *) p_;
+     P *pln;
+     int n;
+     int is, os;
+     plan *cld1 = (plan *) 0;
+     plan *cld2 = (plan *) 0;
+     plan *cld_omega = (plan *) 0;
+     R *buf = (R *) 0;
+     R *O;
+     problem *cldp;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+	  return (plan *) 0;
+
+     n = p->sz->dims[0].n;
+     is = p->sz->dims[0].is;
+     os = p->sz->dims[0].os;
+     O = p->O;
+
+     /* initial allocation for the purpose of planning */
+     buf = (R *) MALLOC(sizeof(R) * (n - 1), BUFFERS);
+
+     cld1 = X(mkplan_d)(plnr, 
+			X(mkproblem_rdft_1_d)(X(mktensor_1d)(n - 1, 1, os),
+					      X(mktensor_1d)(1, 0, 0),
+					      buf, 
+					      O + os,
+					      R2HC));
+     if (!cld1) goto nada;
+
+     cldp =
+          X(mkproblem_rdft_1_d)(
+               X(mktensor_1d)(n - 1, os, 1),
+               X(mktensor_1d)(1, 0, 0),
+	       O + os,
+	       buf, 
+#if R2HC_ONLY_CONV
+	       R2HC
+#else
+	       HC2R
+#endif
+	       );
+     if (!(cld2 = X(mkplan_d)(plnr, cldp))) goto nada;
+
+
+     /* plan for omega */
+     plnr->planner_flags |= ESTIMATE;
+     cld_omega = X(mkplan_d)(plnr, 
+			     X(mkproblem_rdft_1_d)(X(mktensor_1d)(n - 1, 1, 1),
+						   X(mktensor_1d)(1, 0, 0),
+						   buf, buf, R2HC));
+     if (!cld_omega) goto nada;
+
+     /* deallocate buffers; let awake() or apply() allocate them for real */
+     X(ifree)(buf);
+     buf = 0;
+
+     pln = MKPLAN_RDFT(P, &padt, apply);
+     pln->cld1 = cld1;
+     pln->cld2 = cld2;
+     pln->cld_omega = cld_omega;
+     pln->omega = 0;
+     pln->n = n;
+     pln->is = is;
+     pln->os = os;
+     pln->g = X(find_generator)(n);
+     pln->ginv = X(power_mod)(pln->g, n - 2, n);
+     A(MULMOD(pln->g, pln->ginv, n) == 1);
+
+     X(ops_add)(&cld1->ops, &cld2->ops, &pln->super.super.ops);
+     pln->super.super.ops.other += (n - 3) * 3 + (n - 2) * 2 + 5;
+     pln->super.super.ops.add += (n - 3) * 1;
+     pln->super.super.ops.mul += (n - 3) * 2 + 2;
+#if R2HC_ONLY_CONV
+     pln->super.super.ops.other += (n - 2) + 4;
+     pln->super.super.ops.add += (n - 3) * 1 + (n - 2) * 1;
+#endif
+
+     return &(pln->super.super);
+
+ nada:
+     X(ifree0)(buf);
+     X(plan_destroy_internal)(cld_omega);
+     X(plan_destroy_internal)(cld2);
+     X(plan_destroy_internal)(cld1);
+     return 0;
+}
+
+/* constructors */
+
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(dht_rader_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/rdft/direct2.c b/src/fftw3/rdft/direct2.c
new file mode 100644
index 0000000..29a1394
--- /dev/null
+++ b/src/fftw3/rdft/direct2.c
@@ -0,0 +1,216 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: direct2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* direct RDFT2 R2HC/HC2R solver, if we have a codelet */
+
+#include "rdft.h"
+
+typedef union {
+     kr2hc r2hc;
+     khc2r hc2r;
+} kodelet;
+
+typedef struct {
+     solver super;
+     union {
+	  const kr2hc_desc *r2hc;
+	  const khc2r_desc *hc2r;
+     } desc;
+     kodelet k;
+     int sz;
+     rdft_kind kind;
+     const char *nam;
+} S;
+
+typedef struct {
+     plan_rdft2 super;
+
+     stride is, os;
+     int vl;
+     int ivs, ovs;
+     kodelet k;
+     const S *slv;
+     int ilast;
+} P;
+
+static void apply_r2hc(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+     int i, vl = ego->vl, ovs = ego->ovs;
+     ASSERT_ALIGNED_DOUBLE;
+     ego->k.r2hc(r, rio, iio, ego->is, ego->os, ego->os,
+		 vl, ego->ivs, ovs);
+     for (i = 0; i < vl; ++i, iio += ovs)
+	  iio[0] = iio[ego->ilast] = 0;
+}
+
+static void apply_hc2r(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+     ASSERT_ALIGNED_DOUBLE;
+     ego->k.hc2r(rio, iio, r, ego->os, ego->os, ego->is,
+		 ego->vl, ego->ivs, ego->ovs);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(stride_destroy)(ego->is);
+     X(stride_destroy)(ego->os);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     const S *s = ego->slv;
+
+     p->print(p, "(rdft2-%s-direct-%d%v \"%s\")", 
+	      X(rdft_kind_str)(s->kind), s->sz, ego->vl, s->nam);
+}
+
+static int applicable(const solver *ego_, const problem *p_)
+{
+     if (RDFT2P(p_)) {
+          const S *ego = (const S *) ego_;
+          const problem_rdft2 *p = (const problem_rdft2 *) p_;
+	  int vl;
+	  int ivs, ovs;
+
+          return (
+	       1
+	       && p->sz->rnk == 1
+	       && p->vecsz->rnk <= 1
+	       && p->sz->dims[0].n == ego->sz
+	       && p->kind == ego->kind
+
+	       /* check strides etc */
+	       && X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs)
+
+	       && (ego->kind != R2HC ||
+		   ego->desc.r2hc->genus->okp(ego->desc.r2hc, 
+					      p->r, p->rio, p->rio,
+					      p->sz->dims[0].is,
+					      p->sz->dims[0].os,
+					      p->sz->dims[0].os,
+					      vl, ivs, ovs))
+	       && (ego->kind != HC2R ||
+		   ego->desc.hc2r->genus->okp(ego->desc.hc2r,
+					      p->rio, p->rio, p->r,
+					      p->sz->dims[0].is,
+					      p->sz->dims[0].is,
+					      p->sz->dims[0].os,
+					      vl, ivs, ovs))
+	       
+	       && (0
+		   /* can operate out-of-place */
+		   || p->r != p->rio
+
+		   /*
+		    * can compute one transform in-place, no matter
+		    * what the strides are.
+		    */
+		   || p->vecsz->rnk == 0
+
+		   /* can operate in-place as long as strides are the same */
+		   || X(rdft2_inplace_strides)(p, RNK_MINFTY)
+		    )
+	       );
+     }
+
+     return 0;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     P *pln;
+     const problem_rdft2 *p;
+     iodim d;
+     int r2hc_kindp;
+
+     static const plan_adt padt = {
+	  X(rdft2_solve), X(null_awake), print, destroy
+     };
+
+     UNUSED(plnr);
+
+     if (!applicable(ego_, p_))
+          return (plan *)0;
+
+     p = (const problem_rdft2 *) p_;
+
+     r2hc_kindp = p->kind == R2HC;
+     A(r2hc_kindp || p->kind == HC2R);
+
+     pln = MKPLAN_RDFT2(P, &padt, r2hc_kindp ? apply_r2hc : apply_hc2r);
+
+     d = p->sz->dims[0];
+
+     pln->k = ego->k;
+
+     pln->is = X(mkstride)(ego->sz, r2hc_kindp ? d.is : d.os);
+     pln->os = X(mkstride)(d.n/2 + 1, r2hc_kindp ? d.os : d.is);
+
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+
+     pln->ilast = (d.n % 2) ? 0 : (d.n/2) * d.os; /* Nyquist freq., if any */
+
+     pln->slv = ego;
+     X(ops_zero)(&pln->super.super.ops);
+     if (r2hc_kindp)
+	  X(ops_madd2)(pln->vl / ego->desc.r2hc->genus->vl,
+		       &ego->desc.r2hc->ops,
+		       &pln->super.super.ops);
+     else {
+	  X(ops_madd2)(pln->vl / ego->desc.hc2r->genus->vl,
+		       &ego->desc.hc2r->ops,
+		       &pln->super.super.ops);
+	  pln->super.super.ops.other += 2 * pln->vl; /* + 2 stores */
+     }
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+solver *X(mksolver_rdft2_r2hc_direct)(kr2hc k, const kr2hc_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->k.r2hc = k;
+     slv->desc.r2hc = desc;
+     slv->sz = desc->sz;
+     slv->nam = desc->nam;
+     slv->kind = desc->genus->kind;
+     return &(slv->super);
+}
+
+solver *X(mksolver_rdft2_hc2r_direct)(khc2r k, const khc2r_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->k.hc2r = k;
+     slv->desc.hc2r = desc;
+     slv->sz = desc->sz;
+     slv->nam = desc->nam;
+     slv->kind = desc->genus->kind;
+     return &(slv->super);
+}
diff --git a/src/fftw3/rdft/hc2hc-buf.c b/src/fftw3/rdft/hc2hc-buf.c
new file mode 100644
index 0000000..75f761c
--- /dev/null
+++ b/src/fftw3/rdft/hc2hc-buf.c
@@ -0,0 +1,245 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: hc2hc-buf.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* decimation in time Cooley-Tukey */
+#include "rdft.h"
+#include "hc2hc.h"
+
+/*
+   Copy A -> B, where A and B are n0 x n1 complex matrices
+   such that the (i0, i1) element has index (i0 * s0 + i1 * s1).
+   The imaginary strides are of opposite signs to the real strides.
+*/
+static void cpy(int n0, int n1,
+                const R *rA, const R *iA, int sa0, int sa1,
+                R *rB, R *iB, int sb0, int sb1)
+{
+     int i0, i1;
+
+     for (i0 = 0; i0 < n0; ++i0) {
+          const R *pra, *pia;
+          R *prb, *pib;
+          pra = rA; rA += sa0;
+          pia = iA; iA -= sa0;
+          prb = rB; rB += sb0;
+          pib = iB; iB -= sb0;
+
+          for (i1 = 0; i1 < n1; ++i1) {
+               R xr, xi;
+               xr = *pra; pra += sa1;
+               xi = *pia; pia -= sa1;
+               *prb = xr; prb += sb1;
+               *pib = xi; pib -= sb1;
+          }
+     }
+}
+
+static const R *doit(khc2hc k, R *rA, R *iA, const R *W, int ios, int dist,
+                     int r, int batchsz, R *buf, stride bufstride)
+{
+     cpy(r, batchsz, rA, iA, ios, dist, buf, buf + 2*batchsz*r-1, 1, r);
+     W = k(buf, buf + 2*batchsz*r-1, W, bufstride, 2*batchsz + 1, r);
+     cpy(r, batchsz, buf, buf + 2*batchsz*r-1, 1, r, rA, iA, ios, dist);
+     return W;
+}
+
+#define BATCHSZ 4 /* FIXME: parametrize? */
+
+static void apply_dit(const plan *ego_, R *I, R *O)
+{
+     const plan_hc2hc *ego = (const plan_hc2hc *) ego_;
+
+     /* two-dimensional r x vl sub-transform: */
+     {
+	  plan_rdft *cld = (plan_rdft *) ego->cld;
+	  cld->apply((plan *) cld, I, O);
+     }
+
+     {
+          plan_rdft *cld0 = (plan_rdft *) ego->cld0;
+          plan_rdft *cldm = (plan_rdft *) ego->cldm;
+          int i, j, r = ego->r, m = ego->m, vl = ego->vl;
+          int os = ego->os, ovs = ego->ovs, ios = ego->iios;
+	  R *buf;
+
+	  STACK_MALLOC(R *, buf, r * BATCHSZ * 2 * sizeof(R));
+
+          for (i = 0; i < vl; ++i, O += ovs) {
+	       R *rA, *iA;
+	       const R *W;
+
+	       cld0->apply((plan *) cld0, O, O);
+	       
+	       rA = O + os; iA = O + (r * m - 1) * os;
+	       W = ego->W;
+	       for (j = (m-1)/2; j >= BATCHSZ; j -= BATCHSZ) {
+		    W = doit(ego->k, rA, iA, W, ios, os, r, BATCHSZ, buf,
+			     ego->vs);
+		    rA += os * (int)BATCHSZ;
+		    iA -= os * (int)BATCHSZ;
+	       }
+	       /* do remaining j calls, if any */
+               if (j > 0)
+                    doit(ego->k, rA, iA, W, ios, os, r, j, buf, ego->vs);
+
+	       cldm->apply((plan *) cldm, O + os*(m/2), O + os*(m/2));
+	  }
+
+	  STACK_FREE(buf);
+     }
+}
+
+static void apply_dif(const plan *ego_, R *I, R *O)
+{
+     const plan_hc2hc *ego = (const plan_hc2hc *) ego_;
+     R *I0 = I;
+
+     {
+          plan_rdft *cld0 = (plan_rdft *) ego->cld0;
+          plan_rdft *cldm = (plan_rdft *) ego->cldm;
+          int i, j, r = ego->r, m = ego->m, vl = ego->vl;
+          int is = ego->is, ivs = ego->ivs, ios = ego->iios;
+	  R *buf;
+
+	  STACK_MALLOC(R *, buf, r * BATCHSZ * 2 * sizeof(R));
+
+          for (i = 0; i < vl; ++i, I += ivs) {
+	       R *rA, *iA;
+	       const R *W;
+
+	       cld0->apply((plan *) cld0, I, I);
+	       
+	       rA = I + is; iA = I + (r * m - 1) * is;
+	       W = ego->W;
+	       for (j = (m-1)/2; j >= BATCHSZ; j -= BATCHSZ) {
+		    W = doit(ego->k, rA, iA, W, ios, is, r, BATCHSZ, buf,
+			     ego->vs);
+		    rA += is * (int)BATCHSZ;
+		    iA -= is * (int)BATCHSZ;
+	       }
+	       /* do remaining j calls, if any */
+               if (j > 0)
+                    doit(ego->k, rA, iA, W, ios, is, r, j, buf, ego->vs);
+
+	       cldm->apply((plan *) cldm, I + is*(m/2), I + is*(m/2));
+	  }
+
+	  STACK_FREE(buf);
+     }
+
+     /* two-dimensional r x vl sub-transform: */
+     {
+	  plan_rdft *cld = (plan_rdft *) ego->cld;
+	  cld->apply((plan *) cld, I0, O);
+     }
+}
+
+static int applicable0(const solver_hc2hc *ego, const problem *p_,
+		       const planner *plnr)
+{
+     if (X(rdft_hc2hc_applicable)(ego, p_)) {
+          const hc2hc_desc *e = ego->desc;
+          const problem_rdft *p = (const problem_rdft *) p_;
+          iodim *d = p->sz->dims;
+	  int r = e->radix, m = d[0].n / e->radix;
+          return (1
+		  && (p->kind[0]==R2HC || p->I == p->O || DESTROY_INPUTP(plnr))
+                  /* check both batch size and remainder */
+                  && (m < BATCHSZ ||
+                      (e->genus->okp(e, 0, ((const R *)0)+2*BATCHSZ*r-1, 1,0, 
+				     2*BATCHSZ + 1, r)))
+                  && (m < BATCHSZ ||
+                      (e->genus->okp(e, 0, ((const R *)0) 
+				     + 2*(((m-1)/2) % BATCHSZ)*r-1, 1, 0, 
+				     2*(((m-1)/2) % BATCHSZ) + 1, r)))
+	       );
+     }
+     return 0;
+}
+
+static int applicable(const solver_hc2hc *ego, const problem *p_,
+		      const planner *plnr)
+{
+     const problem_rdft *p;
+
+     if (!applicable0(ego, p_, plnr)) return 0;
+
+     p = (const problem_rdft *) p_;
+
+     /* emulate fftw2 behavior */
+     if (NO_VRECURSEP(plnr) && (p->vecsz->rnk > 0)) return 0;
+
+     if (NO_UGLYP(plnr) && 
+	 X(ct_uglyp)(512, p->sz->dims[0].n, ego->desc->radix))
+	  return 0;
+
+     return 1;
+}
+
+static void finish(plan_hc2hc *ego)
+{
+     const hc2hc_desc *d = ego->slv->desc;
+     opcnt t;
+
+     ego->iios = ego->m * (R2HC_KINDP(d->genus->kind) ? ego->os : ego->is);
+     ego->vs = X(mkstride)(ego->r, 1);
+
+     X(ops_add)(&ego->cld0->ops, &ego->cldm->ops, &t);
+     X(ops_madd)(ego->vl, &t, &ego->cld->ops, &ego->super.super.ops);
+     ego->super.super.ops.other += 4 * ego->r * ((ego->m - 1)/2) * ego->vl;
+     X(ops_madd2)(ego->vl * ((ego->m - 1)/2) / d->genus->vl, &d->ops,
+		  &ego->super.super.ops);
+}
+
+static plan *mkplan_ditbuf(const solver *ego, const problem *p, planner *plnr)
+{
+     static const hc2hcadt adt = {
+	  sizeof(plan_hc2hc), 
+	  X(rdft_mkcldrn_dit), finish, applicable, apply_dit
+     };
+     return X(mkplan_rdft_hc2hc)((const solver_hc2hc *) ego, p, plnr, &adt);
+}
+
+solver *X(mksolver_rdft_hc2hc_ditbuf)(khc2hc codelet, const hc2hc_desc *desc)
+{
+     static const solver_adt sadt = { mkplan_ditbuf };
+     static const char name[] = "rdft-ditbuf";
+
+     return X(mksolver_rdft_hc2hc)(codelet, desc, name, &sadt);
+}
+
+static plan *mkplan_difbuf(const solver *ego, const problem *p, planner *plnr)
+{
+     static const hc2hcadt adt = {
+	  sizeof(plan_hc2hc), 
+	  X(rdft_mkcldrn_dif), finish, applicable, apply_dif
+     };
+     return X(mkplan_rdft_hc2hc)((const solver_hc2hc *) ego, p, plnr, &adt);
+}
+
+solver *X(mksolver_rdft_hc2hc_difbuf)(khc2hc codelet, const hc2hc_desc *desc)
+{
+     static const solver_adt sadt = { mkplan_difbuf };
+     static const char name[] = "rdft-difbuf";
+
+     return X(mksolver_rdft_hc2hc)(codelet, desc, name, &sadt);
+}
diff --git a/src/fftw3/rdft/hc2hc-dif.c b/src/fftw3/rdft/hc2hc-dif.c
new file mode 100644
index 0000000..2bcdca2
--- /dev/null
+++ b/src/fftw3/rdft/hc2hc-dif.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: hc2hc-dif.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* decimation in frequency Cooley-Tukey */
+#include "rdft.h"
+#include "hc2hc.h"
+
+static void apply(const plan *ego_, R *I, R *O)
+{
+     const plan_hc2hc *ego = (const plan_hc2hc *) ego_;
+     R *I0 = I;
+
+     {
+          plan_rdft *cld0 = (plan_rdft *) ego->cld0;
+          plan_rdft *cldm = (plan_rdft *) ego->cldm;
+          int i, r = ego->r, m = ego->m, vl = ego->vl;
+          int is = ego->is, ivs = ego->ivs;
+	  
+          for (i = 0; i < vl; ++i, I += ivs) {
+	       cld0->apply((plan *) cld0, I, I);
+               ego->k(I + is, I + (r * m - 1) * is, ego->W, ego->ios, m, is);
+	       cldm->apply((plan *) cldm, I + is*(m/2), I + is*(m/2));
+	  }
+     }
+
+     /* two-dimensional r x vl sub-transform: */
+     {
+	  plan_rdft *cld = (plan_rdft *) ego->cld;
+	  cld->apply((plan *) cld, I0, O);
+     }
+}
+
+static int applicable0(const solver_hc2hc *ego, const problem *p_,
+		       const planner *plnr)
+{
+     if (X(rdft_hc2hc_applicable)(ego, p_)) {
+	  int ivs, ovs;
+	  int vl;
+          const hc2hc_desc *e = ego->desc;
+          const problem_rdft *p = (const problem_rdft *) p_;
+          iodim *d = p->sz->dims;
+	  int m = d[0].n / e->radix;
+	  X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs);
+          return (1
+		  && (p->I == p->O || DESTROY_INPUTP(plnr))
+		  && (e->genus->okp(e, p->I + d[0].is,
+				    p->I + (e->radix * m - 1) * d[0].is, 
+				    (int)m * d[0].is, 0, m, d[0].is))
+		  && (e->genus->okp(e, p->I + ivs + d[0].is,
+				    p->I + ivs + (e->radix * m - 1) * d[0].is, 
+				    (int)m * d[0].is, 0, m, d[0].is))
+	       );
+     }
+     return 0;
+}
+
+static int applicable(const solver_hc2hc *ego, const problem *p_,
+		      const planner *plnr)
+{
+     const problem_rdft *p;
+
+     if (!applicable0(ego, p_, plnr)) return 0;
+
+     p = (const problem_rdft *) p_;
+
+     /* emulate fftw2 behavior */
+     if (NO_VRECURSEP(plnr) && (p->vecsz->rnk > 0)) return 0;
+
+     if (NO_UGLYP(plnr)) {
+	  if (X(ct_uglyp)(16, p->sz->dims[0].n, ego->desc->radix)) return 0;
+	  if (NONTHREADED_ICKYP(plnr)) return 0; /* prefer threaded version */
+     }
+     return 1;
+}
+
+static void finish(plan_hc2hc *ego)
+{
+     const hc2hc_desc *d = ego->slv->desc;
+     opcnt t;
+
+     ego->ios = X(mkstride)(ego->r, ego->m * ego->is);
+
+     X(ops_add)(&ego->cld0->ops, &ego->cldm->ops, &t);
+     X(ops_madd)(ego->vl, &t, &ego->cld->ops, &ego->super.super.ops);
+     X(ops_madd2)(ego->vl * ((ego->m - 1)/2) / d->genus->vl, &d->ops,
+		  &ego->super.super.ops);
+}
+
+static plan *mkplan(const solver *ego, const problem *p, planner *plnr)
+{
+     static const hc2hcadt adt = {
+	  sizeof(plan_hc2hc), 
+	  X(rdft_mkcldrn_dif), finish, applicable, apply
+     };
+     return X(mkplan_rdft_hc2hc)((const solver_hc2hc *) ego, p, plnr, &adt);
+}
+
+
+solver *X(mksolver_rdft_hc2hc_dif)(khc2hc codelet, const hc2hc_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     static const char name[] = "rdft-dif";
+
+     return X(mksolver_rdft_hc2hc)(codelet, desc, name, &sadt);
+}
diff --git a/src/fftw3/rdft/hc2hc-dit.c b/src/fftw3/rdft/hc2hc-dit.c
new file mode 100644
index 0000000..04aa776
--- /dev/null
+++ b/src/fftw3/rdft/hc2hc-dit.c
@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: hc2hc-dit.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* decimation in time Cooley-Tukey */
+#include "rdft.h"
+#include "hc2hc.h"
+
+static void apply(const plan *ego_, R *I, R *O)
+{
+     const plan_hc2hc *ego = (const plan_hc2hc *) ego_;
+
+     /* two-dimensional r x vl sub-transform: */
+     {
+	  plan_rdft *cld = (plan_rdft *) ego->cld;
+	  cld->apply((plan *) cld, I, O);
+     }
+
+     {
+          plan_rdft *cld0 = (plan_rdft *) ego->cld0;
+          plan_rdft *cldm = (plan_rdft *) ego->cldm;
+          int i, r = ego->r, m = ego->m, vl = ego->vl;
+          int os = ego->os, ovs = ego->ovs;
+
+          for (i = 0; i < vl; ++i, O += ovs) {
+	       cld0->apply((plan *) cld0, O, O);
+               ego->k(O + os, O + (r * m - 1) * os, ego->W, ego->ios, m, os);
+	       cldm->apply((plan *) cldm, O + os*(m/2), O + os*(m/2));
+	  }
+     }
+}
+
+static int applicable0(const solver_hc2hc *ego, const problem *p_,
+		       const planner *plnr)
+{
+     UNUSED(plnr);
+     if (X(rdft_hc2hc_applicable)(ego, p_)) {
+	  int ivs, ovs;
+	  int vl;
+          const hc2hc_desc *e = ego->desc;
+          const problem_rdft *p = (const problem_rdft *) p_;
+          iodim *d = p->sz->dims;
+	  int m = d[0].n / e->radix;
+	  X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs);
+          return (1
+		  && (e->genus->okp(e, p->O + d[0].os,
+				    p->O + (e->radix * m - 1) * d[0].os, 
+				    (int)m * d[0].os, 0, m, d[0].os))
+		  && (e->genus->okp(e, p->O + ovs + d[0].os,
+				    p->O + ovs + (e->radix * m - 1) * d[0].os, 
+				    (int)m * d[0].os, 0, m, d[0].os))
+	       );
+     }
+     return 0;
+}
+
+static int applicable(const solver_hc2hc *ego, const problem *p_,
+		      const planner *plnr)
+{
+     const problem_rdft *p;
+
+     if (!applicable0(ego, p_, plnr)) return 0;
+
+     p = (const problem_rdft *) p_;
+
+     /* emulate fftw2 behavior */
+     if (NO_VRECURSEP(plnr) && (p->vecsz->rnk > 0)) return 0;
+
+     if (NO_UGLYP(plnr)) {
+	  if (X(ct_uglyp)(16, p->sz->dims[0].n, ego->desc->radix)) return 0;
+	  if (NONTHREADED_ICKYP(plnr))
+	       return 0; /* prefer threaded version */
+     }
+
+     return 1;
+}
+
+static void finish(plan_hc2hc *ego)
+{
+     const hc2hc_desc *d = ego->slv->desc;
+     opcnt t;
+
+     ego->ios = X(mkstride)(ego->r, ego->m * ego->os);
+
+     X(ops_add)(&ego->cld0->ops, &ego->cldm->ops, &t);
+     X(ops_madd)(ego->vl, &t, &ego->cld->ops, &ego->super.super.ops);
+     X(ops_madd2)(ego->vl * ((ego->m - 1)/2) / d->genus->vl, &d->ops,
+		  &ego->super.super.ops);
+}
+
+
+static plan *mkplan(const solver *ego, const problem *p, planner *plnr)
+{
+     static const hc2hcadt adt = {
+	  sizeof(plan_hc2hc), 
+	  X(rdft_mkcldrn_dit), finish, applicable, apply
+     };
+     return X(mkplan_rdft_hc2hc)((const solver_hc2hc *) ego, p, plnr, &adt);
+}
+
+
+solver *X(mksolver_rdft_hc2hc_dit)(khc2hc codelet, const hc2hc_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     static const char name[] = "rdft-dit";
+
+     return X(mksolver_rdft_hc2hc)(codelet, desc, name, &sadt);
+}
diff --git a/src/fftw3/rdft/hc2hc.c b/src/fftw3/rdft/hc2hc.c
new file mode 100644
index 0000000..926273c
--- /dev/null
+++ b/src/fftw3/rdft/hc2hc.c
@@ -0,0 +1,222 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: hc2hc.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* generic Cooley-Tukey routines */
+#include "rdft.h"
+#include "hc2hc.h"
+
+static void destroy(plan *ego_)
+{
+     plan_hc2hc *ego = (plan_hc2hc *) ego_;
+
+     X(plan_destroy_internal)(ego->cld);
+     X(plan_destroy_internal)(ego->cld0);
+     X(plan_destroy_internal)(ego->cldm);
+     X(stride_destroy)(ego->ios);
+     X(stride_destroy)(ego->vs);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     plan_hc2hc *ego = (plan_hc2hc *) ego_;
+
+     AWAKE(ego->cld, flg);
+     AWAKE(ego->cld0, flg);
+     AWAKE(ego->cldm, flg);
+
+     if (flg) {
+	  const tw_instr *tw = ego->slv->desc->tw;
+	  X(mktwiddle)(&ego->td, tw, ego->n, ego->r, (ego->m + 1) / 2);
+	  /* 0th twiddle is handled by cld0: */
+	  ego->W = ego->td->W + X(twiddle_length)(ego->r, tw);
+     } else {
+	  X(twiddle_destroy)(&ego->td);
+          ego->W = 0;
+     }
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const plan_hc2hc *ego = (const plan_hc2hc *) ego_;
+     const solver_hc2hc *slv = ego->slv;
+     const hc2hc_desc *e = slv->desc;
+
+     p->print(p, "(%s-%d/%d%v \"%s\"%(%p%)%(%p%)%(%p%))",
+              slv->nam, ego->r, X(twiddle_length)(ego->r, e->tw),
+	      ego->vl, e->nam, ego->cld0, ego->cldm, ego->cld);
+}
+
+#define divides(a, b) (((int)(b) % (int)(a)) == 0)
+
+int X(rdft_hc2hc_applicable)(const solver_hc2hc *ego, const problem *p_)
+{
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          const hc2hc_desc *d = ego->desc;
+          return (1
+                  && p->sz->rnk == 1
+                  && p->vecsz->rnk <= 1
+		  && p->kind[0] == d->genus->kind
+                  && divides(d->radix, p->sz->dims[0].n)
+		  && d->radix < p->sz->dims[0].n /* avoid inf. loops in cld0 */
+	       );
+     }
+     return 0;
+}
+
+
+static const plan_adt padt =
+{
+     X(rdft_solve),
+     awake,
+     print,
+     destroy
+};
+
+plan *X(mkplan_rdft_hc2hc)(const solver_hc2hc *ego,
+			   const problem *p_,
+			   planner *plnr,
+			   const hc2hcadt *adt)
+{
+     plan_hc2hc *pln;
+     plan *cld = 0, *cld0 = 0, *cldm = 0;
+     int n, r, m;
+     problem *cldp = 0, *cld0p = 0, *cldmp = 0;
+     iodim *d;
+     const problem_rdft *p;
+     const hc2hc_desc *e = ego->desc;
+
+     if (!adt->applicable(ego, p_, plnr))
+          return (plan *) 0;
+
+     p = (const problem_rdft *) p_;
+     d = p->sz->dims;
+     n = d[0].n;
+     r = e->radix;
+     m = n / r;
+
+     adt->mkcldrn(ego, p, &cldp, &cld0p, &cldmp);
+
+     cld = X(mkplan_d)(plnr, cldp); cldp = 0;
+     if (!cld) goto nada;
+
+     cld0 = X(mkplan_d)(plnr, cld0p); cld0p = 0;
+     if (!cld0) goto nada;
+
+     cldm = X(mkplan_d)(plnr, cldmp); cldmp = 0;
+     if (!cldm) goto nada;
+
+     A(adt->pln_size >= sizeof(plan_hc2hc));
+     pln = (plan_hc2hc *) X(mkplan_rdft)(adt->pln_size, &padt, adt->apply);
+
+     pln->slv = ego;
+     pln->cld = cld;
+     pln->cld0 = cld0;
+     pln->cldm = cldm;
+     pln->k = ego->k;
+     pln->n = n;
+     pln->r = r;
+     pln->m = m;
+
+     pln->is = d[0].is;
+     pln->os = d[0].os;
+
+     pln->ios = pln->vs = 0;
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     pln->td = 0;
+     adt->finish(pln);
+
+     return &(pln->super.super);
+
+ nada:
+     X(problem_destroy)(cldmp);
+     X(problem_destroy)(cld0p);
+     X(problem_destroy)(cldp);
+     X(plan_destroy_internal)(cldm);
+     X(plan_destroy_internal)(cld0);
+     X(plan_destroy_internal)(cld);
+     return (plan *) 0;
+}
+
+solver *X(mksolver_rdft_hc2hc)(khc2hc k, const hc2hc_desc *desc,
+			       const char *nam, const solver_adt *adt)
+{
+     solver_hc2hc *slv;
+
+     slv = MKSOLVER(solver_hc2hc, adt);
+
+     slv->desc = desc;
+     slv->k = k;
+     slv->nam = nam;
+     return &(slv->super);
+}
+
+/* routines to create children are shared by many solvers */
+
+void X(rdft_mkcldrn_dit)(const solver_hc2hc *ego, const problem_rdft *p,
+                         problem **cldp, problem **cld0p, problem **cldmp)
+{
+     iodim *d = p->sz->dims;
+     const hc2hc_desc *e = ego->desc;
+     int m = d[0].n / e->radix;
+     int omid = d[0].os * (m/2);
+
+     tensor *null, *radix = X(mktensor_1d)(e->radix, d[0].is, m * d[0].os);
+     tensor *cld_vec = X(tensor_append)(radix, p->vecsz);
+     X(tensor_destroy)(radix);
+     A(p->kind[0] == R2HC);
+
+     *cldp = X(mkproblem_rdft_d)(X(mktensor_1d)(m, e->radix*d[0].is, d[0].os),
+				 cld_vec, p->I, p->O, p->kind);
+
+     radix = X(mktensor_1d)(e->radix, m * d[0].os, m * d[0].os);
+     null = X(mktensor_0d)();
+     *cld0p = X(mkproblem_rdft_1)(radix, null, p->O, p->O, R2HC);
+     *cldmp = X(mkproblem_rdft_1)(m%2 ? null : radix, null,
+				  p->O + omid, p->O + omid, R2HCII);
+     X(tensor_destroy2)(null, radix);
+}
+
+
+void X(rdft_mkcldrn_dif)(const solver_hc2hc *ego, const problem_rdft *p,
+                         problem **cldp, problem **cld0p, problem **cldmp)
+{
+     iodim *d = p->sz->dims;
+     const hc2hc_desc *e = ego->desc;
+     int m = d[0].n / e->radix;
+     int imid = d[0].is * (m/2);
+
+     tensor *null, *radix = X(mktensor_1d)(e->radix, m * d[0].is, d[0].os);
+     tensor *cld_vec = X(tensor_append)(radix, p->vecsz);
+     X(tensor_destroy)(radix);
+     A(p->kind[0] == HC2R);
+
+     *cldp = X(mkproblem_rdft_d)(X(mktensor_1d)(m, d[0].is, e->radix*d[0].os),
+				 cld_vec, p->I, p->O, p->kind);
+
+     radix = X(mktensor_1d)(e->radix, m * d[0].is, m * d[0].is);
+     null = X(mktensor_0d)();
+     *cld0p = X(mkproblem_rdft_1)(radix, null, p->I, p->I, HC2R);
+     *cldmp = X(mkproblem_rdft_1)(m%2 ? null : radix, null, 
+				  p->I + imid, p->I + imid, HC2RIII);
+     X(tensor_destroy2)(null, radix);
+}
diff --git a/src/fftw3/rdft/hc2hc.h b/src/fftw3/rdft/hc2hc.h
new file mode 100644
index 0000000..6c062cd
--- /dev/null
+++ b/src/fftw3/rdft/hc2hc.h
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+typedef struct {
+     solver super;
+     const char *nam;
+     const hc2hc_desc *desc;
+     khc2hc k;
+} solver_hc2hc;
+
+typedef struct {
+     plan_rdft super;
+     khc2hc k;
+     plan *cld0, *cldm; /* children for 0th and middle butterflies */
+     plan *cld;
+     R *W;
+     int n, r, m, vl;
+     int is, os, ivs, ovs, iios;
+     stride ios, vs;
+     const solver_hc2hc *slv;
+     twid *td;
+} plan_hc2hc;
+
+/* data type describing a generic Cooley-Tukey solver */
+typedef struct
+{
+     size_t pln_size;
+     void (*mkcldrn)(const solver_hc2hc *, const problem_rdft *p,
+		     problem **cldp, problem **cld0p, problem **cldmp);
+     void (*finish)(plan_hc2hc *ego);
+     int (*applicable)(const solver_hc2hc *ego, const problem *p,
+		       const planner *plnr);
+     rdftapply apply;
+} hc2hcadt;
+
+int X(rdft_hc2hc_applicable)(const solver_hc2hc *ego, const problem *p_);
+
+plan *X(mkplan_rdft_hc2hc)(const solver_hc2hc *ego,
+			   const problem *p_,
+			   planner *plnr,
+			   const hc2hcadt *adt);
+
+solver *X(mksolver_rdft_hc2hc)(khc2hc k, const hc2hc_desc *desc,
+			       const char *nam, const solver_adt *adt);
+
+void X(rdft_mkcldrn_dit)(const solver_hc2hc *, const problem_rdft *p,
+			 problem **cldp, problem **cld0p, problem **cldmp);
+void X(rdft_mkcldrn_dif)(const solver_hc2hc *, const problem_rdft *p,
+			 problem **cldp, problem **cld0p, problem **cldmp);
diff --git a/src/fftw3/rdft/khc2hc-dif.c b/src/fftw3/rdft/khc2hc-dif.c
new file mode 100644
index 0000000..14806f1
--- /dev/null
+++ b/src/fftw3/rdft/khc2hc-dif.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: khc2hc-dif.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+void (*X(khc2hc_dif_register_hook))(planner *, khc2hc, const hc2hc_desc *)=0;
+
+void X(khc2hc_dif_register)(planner *p, khc2hc codelet, const hc2hc_desc *desc)
+{
+     REGISTER_SOLVER(p, X(mksolver_rdft_hc2hc_dif)(codelet, desc));
+     REGISTER_SOLVER(p, X(mksolver_rdft_hc2hc_difbuf)(codelet, desc));
+     if (X(khc2hc_dif_register_hook))
+	  X(khc2hc_dif_register_hook)(p, codelet, desc);
+}
diff --git a/src/fftw3/rdft/khc2hc-dit.c b/src/fftw3/rdft/khc2hc-dit.c
new file mode 100644
index 0000000..24ba8d3
--- /dev/null
+++ b/src/fftw3/rdft/khc2hc-dit.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: khc2hc-dit.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+void (*X(khc2hc_dit_register_hook))(planner *, khc2hc, const hc2hc_desc *)=0;
+
+void X(khc2hc_dit_register)(planner *p, khc2hc codelet, const hc2hc_desc *desc)
+{
+     REGISTER_SOLVER(p, X(mksolver_rdft_hc2hc_dit)(codelet, desc));
+     REGISTER_SOLVER(p, X(mksolver_rdft_hc2hc_ditbuf)(codelet, desc));
+     if (X(khc2hc_dit_register_hook))
+	  X(khc2hc_dit_register_hook)(p, codelet, desc);
+}
diff --git a/src/fftw3/rdft/khc2r.c b/src/fftw3/rdft/khc2r.c
new file mode 100644
index 0000000..d796bca
--- /dev/null
+++ b/src/fftw3/rdft/khc2r.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: khc2r.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+void X(khc2r_register)(planner *p, khc2r codelet, const khc2r_desc *desc)
+{
+     REGISTER_SOLVER(p, X(mksolver_rdft_hc2r_direct)(codelet, desc));
+     REGISTER_SOLVER(p, X(mksolver_rdft2_hc2r_direct)(codelet, desc));
+}
diff --git a/src/fftw3/rdft/kr2hc.c b/src/fftw3/rdft/kr2hc.c
new file mode 100644
index 0000000..9840186
--- /dev/null
+++ b/src/fftw3/rdft/kr2hc.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: kr2hc.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+void X(kr2hc_register)(planner *p, kr2hc codelet, const kr2hc_desc *desc)
+{
+     REGISTER_SOLVER(p, X(mksolver_rdft_r2hc_direct)(codelet, desc));
+     REGISTER_SOLVER(p, X(mksolver_rdft2_r2hc_direct)(codelet, desc));
+}
diff --git a/src/fftw3/rdft/kr2r.c b/src/fftw3/rdft/kr2r.c
new file mode 100644
index 0000000..31b1c10
--- /dev/null
+++ b/src/fftw3/rdft/kr2r.c
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: kr2r.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+void X(kr2r_register)(planner *p, kr2r codelet, const kr2r_desc *desc)
+{
+     REGISTER_SOLVER(p, X(mksolver_rdft_r2r_direct)(codelet, desc));
+}
diff --git a/src/fftw3/rdft/nop2.c b/src/fftw3/rdft/nop2.c
new file mode 100644
index 0000000..f647b29
--- /dev/null
+++ b/src/fftw3/rdft/nop2.c
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: nop2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* plans for vrank -infty RDFT2s (nothing to do), as well as in-place
+   rank-0 HC2R.  Note that in-place rank-0 R2HC is *not* a no-op, because
+   we have to set the imaginary parts of the output to zero. */
+
+#include "rdft.h"
+
+static void apply(const plan *ego_, R *r, R *rio, R *iio)
+{
+     UNUSED(ego_);
+     UNUSED(r);
+     UNUSED(rio);
+     UNUSED(iio);
+}
+
+static int applicable(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (RDFT2P(p_)) {
+          const problem_rdft2 *p = (const problem_rdft2 *) p_;
+          return(0
+		 /* case 1 : -infty vector rank */
+		 || (p->vecsz->rnk == RNK_MINFTY)
+		 
+		 /* case 2 : rank-0 in-place HC2R rdft */
+		 || (1
+		     && p->kind == HC2R
+		     && p->sz->rnk == 0
+		     && FINITE_RNK(p->vecsz->rnk)
+		     && (p->r == p->rio || p->r == p->iio)
+		     && X(rdft2_inplace_strides)(p, RNK_MINFTY)
+		      ));
+     }
+     return 0;
+}
+
+static void print(const plan *ego, printer *p)
+{
+     UNUSED(ego);
+     p->print(p, "(rdft2-nop)");
+}
+
+static plan *mkplan(const solver *ego, const problem *p, planner *plnr)
+{
+     static const plan_adt padt = {
+	  X(rdft2_solve), X(null_awake), print, X(plan_null_destroy)
+     };
+     plan_rdft2 *pln;
+
+     UNUSED(plnr);
+
+     if (!applicable(ego, p))
+          return (plan *) 0;
+     pln = MKPLAN_RDFT2(plan_rdft2, &padt, apply);
+     X(ops_zero)(&pln->super.ops);
+
+     return &(pln->super);
+}
+
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     return MKSOLVER(solver, &sadt);
+}
+
+void X(rdft2_nop_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/rdft/plan2.c b/src/fftw3/rdft/plan2.c
new file mode 100644
index 0000000..b04c19d
--- /dev/null
+++ b/src/fftw3/rdft/plan2.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: plan2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+plan *X(mkplan_rdft2)(size_t size, const plan_adt *adt, rdft2apply apply)
+{
+     plan_rdft2 *ego;
+
+     ego = (plan_rdft2 *) X(mkplan)(size, adt);
+     ego->apply = apply;
+
+     return &(ego->super);
+}
diff --git a/src/fftw3/rdft/problem2.c b/src/fftw3/rdft/problem2.c
new file mode 100644
index 0000000..57a5764
--- /dev/null
+++ b/src/fftw3/rdft/problem2.c
@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: problem2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "dft.h"
+#include "rdft.h"
+#include <stddef.h>
+
+static void destroy(problem *ego_)
+{
+     problem_rdft2 *ego = (problem_rdft2 *) ego_;
+     X(tensor_destroy2)(ego->vecsz, ego->sz);
+     X(ifree)(ego_);
+}
+
+static void hash(const problem *p_, md5 *m)
+{
+     const problem_rdft2 *p = (const problem_rdft2 *) p_;
+     X(md5puts)(m, "rdft2");
+     X(md5int)(m, p->r == p->rio);
+     X(md5int)(m, p->r == p->iio);
+     X(md5ptrdiff)(m, p->iio - p->rio);
+     X(md5int)(m, X(alignment_of)(p->r));
+     X(md5int)(m, X(alignment_of)(p->rio)); 
+     X(md5int)(m, X(alignment_of)(p->iio)); 
+     X(md5int)(m, p->kind);
+     X(tensor_md5)(m, p->sz);
+     X(tensor_md5)(m, p->vecsz);
+}
+
+static void print(problem *ego_, printer *p)
+{
+     problem_rdft2 *ego = (problem_rdft2 *) ego_;
+     p->print(p, "(rdft2 %d %td %td %d %T %T)", 
+	      X(alignment_of)(ego->r),
+	      ego->rio - ego->r, 
+	      ego->iio - ego->r,
+	      (int)(ego->kind),
+	      ego->sz,
+	      ego->vecsz);
+}
+
+static void zero(const problem *ego_)
+{
+     const problem_rdft2 *ego = (const problem_rdft2 *) ego_;
+     tensor *sz;
+     if (ego->kind == R2HC) {
+	  sz = X(tensor_append)(ego->vecsz, ego->sz);
+	  X(rdft_zerotens)(sz, UNTAINT(ego->r));
+     }
+     else {
+	  tensor *sz2 = X(tensor_copy)(ego->sz);
+	  if (sz2->rnk > 0) /* ~half as many complex outputs */
+	       sz2->dims[0].n = sz2->dims[0].n / 2 + 1;
+	  sz = X(tensor_append)(ego->vecsz, sz2);
+	  X(tensor_destroy)(sz2);
+	  X(dft_zerotens)(sz, UNTAINT(ego->rio), UNTAINT(ego->iio));
+     }
+     X(tensor_destroy)(sz);
+}
+
+static const problem_adt padt =
+{
+     hash,
+     zero,
+     print,
+     destroy
+};
+
+int X(problem_rdft2_p)(const problem *p)
+{
+     return (p->adt == &padt);
+}
+
+problem *X(mkproblem_rdft2)(const tensor *sz, const tensor *vecsz,
+			    R *r, R *rio, R *iio, rdft_kind kind)
+{
+     problem_rdft2 *ego =
+          (problem_rdft2 *)X(mkproblem)(sizeof(problem_rdft2), &padt);
+
+     A(X(tensor_kosherp)(sz));
+     A(X(tensor_kosherp)(vecsz));
+     A(FINITE_RNK(sz->rnk));
+
+     if (UNTAINT(r) == UNTAINT(rio))
+	  r = rio = JOIN_TAINT(r, rio);
+     if (UNTAINT(r) == UNTAINT(iio))
+	  r = iio = JOIN_TAINT(r, iio);
+
+     /* correctness condition: */
+     A(TAINTOF(rio) == TAINTOF(iio));
+
+     if (sz->rnk > 1) { /* have to compress rnk-1 dims separately, ugh */
+	  tensor *szc = X(tensor_copy_except)(sz, sz->rnk - 1);
+	  tensor *szr = X(tensor_copy_sub)(sz, sz->rnk - 1, 1);
+	  tensor *szcc = X(tensor_compress)(szc);
+	  if (szcc->rnk > 0)
+	       ego->sz = X(tensor_append)(szcc, szr);
+	  else
+	       ego->sz = X(tensor_compress)(szr);
+	  X(tensor_destroy2)(szc, szr); X(tensor_destroy)(szcc);
+     }
+     else
+	  ego->sz = X(tensor_compress)(sz);
+     ego->vecsz = X(tensor_compress_contiguous)(vecsz);
+     ego->r = r;
+     ego->rio = rio;
+     ego->iio = iio;
+     ego->kind = kind;
+
+     A(kind == R2HC || kind == HC2R);
+     A(FINITE_RNK(ego->sz->rnk));
+     return &(ego->super);
+}
+
+/* Same as X(mkproblem_rdft2), but also destroy input tensors. */
+problem *X(mkproblem_rdft2_d)(tensor *sz, tensor *vecsz,
+			      R *r, R *rio, R *iio, rdft_kind kind)
+{
+     problem *p;
+     p = X(mkproblem_rdft2)(sz, vecsz, r, rio, iio, kind);
+     X(tensor_destroy2)(vecsz, sz);
+     return p;
+}
diff --git a/src/fftw3/rdft/rader-hc2hc.c b/src/fftw3/rdft/rader-hc2hc.c
new file mode 100644
index 0000000..f1b6f34
--- /dev/null
+++ b/src/fftw3/rdft/rader-hc2hc.c
@@ -0,0 +1,513 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "rdft.h"
+#include "dft.h"
+
+/*
+ * Compute transforms with large prime factors using Rader's trick:
+ * turn the factors into convolutions of size n - 1, which you then
+ * perform via a pair of FFTs.  This file contains only twiddle hc2hc
+ * transforms, which are actually ordinary complex transforms in a
+ * slightly funny order.
+ */
+
+typedef struct {
+     solver super;
+     rdft_kind kind;
+} S;
+
+typedef struct {
+     plan_rdft super;
+
+     plan *cldr, *cldr0;
+     plan *cld;
+     R *W;
+     R *omega;
+     int m, r, g, ginv;
+     int os, ios;
+     rdft_kind kind;
+} P;
+
+static rader_tl *twiddles = 0;
+
+/***************************************************************************/
+
+/* Below, we extensively use the identity that fft(x*)* = ifft(x) in
+   order to share data between forward and backward transforms and to
+   obviate the necessity of having separate forward and backward
+   plans. */
+
+static void apply_aux(int r, plan_dft *cldr, const R *omega,
+		      R *buf, R *ro, R i0, R *io)
+{
+     R r0;
+     int k;
+
+     /* compute DFT of buf, operating in-place */
+     cldr->apply((plan *) cldr, buf, buf+1, buf, buf+1);
+
+     /* set output DC component: */
+     ro[0] = (r0 = ro[0]) + buf[0];
+     io[0] = i0 + buf[1];
+
+     /* now, multiply by omega: */
+     for (k = 0; k < r - 1; ++k) {
+	  R rB, iB, rW, iW;
+	  rW = omega[2*k];
+	  iW = omega[2*k+1];
+	  rB = buf[2*k];
+	  iB = buf[2*k+1];
+	  buf[2*k] = rW * rB - iW * iB;
+	  buf[2*k+1] = -(rW * iB + iW * rB);
+     }
+     
+     /* this will add input[0] to all of the outputs after the ifft */
+     buf[0] += r0;
+     buf[1] -= i0;
+
+     /* inverse FFT: */
+     cldr->apply((plan *) cldr, buf, buf+1, buf, buf+1);
+}
+
+static void apply_dit(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     plan_dft *cldr;
+     int os, ios;
+     int j, k, gpower, g, ginv, r, m;
+     R *buf, *rio, *ii, *io;
+     const R *omega, *W;
+
+     /* size-m child transforms: */
+     {
+	   plan_rdft *cld = (plan_rdft *) ego->cld;
+	   cld->apply((plan *) cld, I, O);
+     }
+
+     /* 0th twiddle transform is just size-r (prime) R2HC: */
+     {
+	   plan_rdft *cldr0 = (plan_rdft *) ego->cldr0;
+	   cldr0->apply((plan *) cldr0, O, O);
+     }
+
+     cldr = (plan_dft *) ego->cldr;
+     r = ego->r;
+     m = ego->m;
+     g = ego->g; 
+     ginv = ego->ginv;
+     omega = ego->omega;
+     W = ego->W;
+     os = ego->os;
+     ios = ego->ios;
+     gpower = 1;
+     rio = O + os;
+     ii = O + (m - 1) * os;
+     io = O + (r * m - 1) * os;
+
+     buf = (R *) MALLOC(sizeof(R) * (r - 1) * 2, BUFFERS);
+
+     for (j = 2; j < m; j += 2, rio += os, ii -= os, io -= os, W += 2*(r-1)) {
+	  /* First, permute the input and multiply by W, storing in buf: */
+	  A(gpower == 1);
+	  for (k = 0; k < r - 1; ++k, gpower = MULMOD(gpower, g, r)) {
+	       R rA, iA, rW, iW;
+	       rA = rio[gpower * ios];
+	       iA = ii[gpower * ios];
+	       rW = W[2*k];
+	       iW = W[2*k+1];
+	       buf[2*k] = rW * rA - iW * iA;
+	       buf[2*k+1] = rW * iA + iW * rA;
+	  }
+	  /* gpower == g^(r-1) mod r == 1 */;
+	  
+	  apply_aux(r, cldr, omega, buf, rio, ii[0], io);
+
+	  /* finally, do inverse permutation to unshuffle the output: */
+	  A(gpower == 1);
+	  for (k = 0; k < r - 1; ++k, gpower = MULMOD(gpower, ginv, r)) {
+	       rio[gpower * ios] = buf[2*k];
+	       io[-gpower * ios] = -buf[2*k+1];
+	  }
+	  A(gpower == 1);
+
+	  /* second half of array must be fiddled to get real/imag
+             parts in correct spots: */
+	  for (k = (r+1)/2; k < r; ++k) {
+	       R t;
+	       t = rio[k * ios];
+	       rio[k * ios] = -io[-k * ios];
+	       io[-k * ios] = t;
+	  }
+     }
+
+     /* Avoid funny m/2-th iter by requiring m odd.  This always
+	happens anyway because all the factors of 2 get divided out
+	first by codelets (Rader is UGLY for small factors). */
+
+     X(ifree)(buf);
+}
+
+static void apply_dif(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     plan_dft *cldr;
+     int is, ios;
+     int j, k, gpower, g, ginv, r, m;
+     R *buf, *rio, *ii, *io;
+     const R *omega, *W;
+
+     /* 0th twiddle transform is just size-r (prime) HC2R: */
+     {
+	   plan_rdft *cldr0 = (plan_rdft *) ego->cldr0;
+	   cldr0->apply((plan *) cldr0, I, I);
+     }
+
+     cldr = (plan_dft *) ego->cldr;
+     r = ego->r;
+     m = ego->m;
+     g = ego->g; 
+     ginv = ego->ginv;
+     omega = ego->omega;
+     W = ego->W + 2*(r-1); /* simplify reverse indexing of W */
+     is = ego->os;
+     ios = ego->ios;
+     gpower = 1;
+     rio = I + is;
+     io = I + (m - 1) * is;
+     ii = I + (r * m - 1) * is;
+
+     buf = (R *) MALLOC(sizeof(R) * (r - 1) * 2, BUFFERS);
+
+     for (j = 2; j < m; j += 2, rio += is, ii -= is, io -= is, W += 2*(r-1)) {
+	  /* second half of array must be unfiddled to get real/imag
+             parts from correct spots: */
+	  for (k = (r+1)/2; k < r; ++k) {
+	       R t;
+	       t = rio[k * ios];
+	       rio[k * ios] = ii[-k * ios];
+	       ii[-k * ios] = -t;
+	  }
+
+	  /* First, permute the input, storing in buf: */
+	  A(gpower == 1);
+	  for (k = 0; k < r - 1; ++k, gpower = MULMOD(gpower, g, r)) {
+	       buf[2*k] = rio[gpower * ios];
+	       buf[2*k+1] = -ii[-gpower * ios];
+	  }
+	  /* gpower == g^(r-1) mod r == 1 */;
+	  A(gpower == 1);
+	  
+	  apply_aux(r, cldr, omega, buf, rio, -ii[0], io);
+	  io[0] = -io[0];
+
+	  /* finally, do inverse permutation to unshuffle the output,
+             also multiplying by the inverse twiddle factors W*.
+	     The twiddle factors are accessed in reverse order W[-k],
+	     because here we exponentiating ginv and not g as in
+	     mktwiddle. */
+	  { /* W[-0] = W[0] case must be handled specially */
+               R rA, iA, rW, iW;
+               rA = buf[0]; iA = buf[1];
+               rW = W[-2*(r-1)]; iW = W[-2*(r-1) + 1];
+               rio[ios] = rA * rW + iA * iW;
+               io[ios] = iA * rW - rA * iW;
+	  }
+	  gpower = ginv;
+	  for (k = 1; k < r - 1; ++k, gpower = MULMOD(gpower, ginv, r)) {
+	       R rA, iA, rW, iW;
+	       rA = buf[2*k]; iA = buf[2*k+1];
+	       rW = W[-2*k]; iW = W[-2*k+1];
+	       rio[gpower * ios] = rA * rW + iA * iW;
+	       io[gpower * ios] = iA * rW - rA * iW;
+	  }
+	  A(gpower == 1);
+     }
+
+     /* Avoid funny m/2-th iter by requiring m odd.  This always
+	happens anyway because all the factors of 2 get divided out
+	first by codelets (Rader is UGLY for small factors). */
+
+     X(ifree)(buf);
+
+     /* size-m child transforms: */
+     {
+	   plan_rdft *cld = (plan_rdft *) ego->cld;
+	   cld->apply((plan *) cld, I, O);
+     }
+}
+
+static R *mktwiddle(int m, int r, int g)
+{
+     int i, j, gpower;
+     int n = r * m;
+     R *W;
+
+     if ((W = X(rader_tl_find)(m, r, g, twiddles)))
+	  return W;
+
+     W = (R *)MALLOC(sizeof(R) * (r - 1) * ((m-1)/2) * 2, TWIDDLES);
+     for (i = 1; i < (m+1)/2; ++i) {
+	  for (gpower = 1, j = 0; j < r - 1;
+	       ++j, gpower = MULMOD(gpower, g, r)) {
+	       int k = (i - 1) * (r - 1) + j;
+	       W[2*k] = X(cos2pi)(i * gpower, n);
+	       W[2*k+1] = FFT_SIGN * X(sin2pi)(i * gpower, n);
+	  }
+	  A(gpower == 1);
+     }
+
+     X(rader_tl_insert)(m, r, g, W, &twiddles);
+     return W;
+}
+
+static void free_twiddle(R *twiddle)
+{
+     X(rader_tl_delete)(twiddle, &twiddles);
+}
+
+/***************************************************************************/
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+
+     AWAKE(ego->cldr0, flg);
+     AWAKE(ego->cldr, flg);
+     AWAKE(ego->cld, flg);
+
+     if (flg) {
+	  if (!ego->omega) 
+	       ego->omega = 
+		    X(dft_rader_mkomega)(ego->cldr, ego->r, ego->ginv);
+	  if (!ego->W)
+	       ego->W = mktwiddle(ego->m, ego->r, ego->g);
+     } else {
+	  X(dft_rader_free_omega)(&ego->omega);
+	  free_twiddle(ego->W);
+	  ego->W = 0;
+     }
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+     X(plan_destroy_internal)(ego->cldr);
+     X(plan_destroy_internal)(ego->cldr0);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+
+     p->print(p, "(rdft-rader-%s-%d%(%p%)%(%p%)%(%p%))",
+	      ego->kind == R2HC ? "r2hc-dit" : "hc2r-dif",
+              ego->r, ego->cldr0, ego->cldr, ego->cld);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     if (RDFTP(p_)) {
+	  const S *ego = (const S *) ego_;
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk == 0
+		  && p->sz->dims[0].n > 1
+		  && p->sz->dims[0].n % 4 /* make sure n / r = m is odd */
+		  && p->kind[0] == ego->kind
+		  && !X(is_prime)(p->sz->dims[0].n) /* avoid inf. loops planning cldr0 */
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego_, const problem *p_, 
+		      const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0(ego_, p_));
+}
+
+static int mkP(P *pln, int r, R *O, int ios, rdft_kind kind, planner *plnr)
+{
+     plan *cldr = (plan *) 0;
+     plan *cldr0 = (plan *) 0;
+     R *buf = (R *) 0;
+
+     cldr0 = X(mkplan_d)(plnr, 
+			 X(mkproblem_rdft_1_d)(X(mktensor_1d)(r, ios, ios),
+					       X(mktensor_1d)(1, 0, 0),
+					       O, O, kind));
+     if (!cldr0) goto nada;
+
+     /* initial allocation for the purpose of planning */
+     buf = (R *) MALLOC(sizeof(R) * (r - 1) * 2, BUFFERS);
+
+     cldr = X(mkplan_d)(plnr, X(mkproblem_dft_d)(X(mktensor_1d)(r - 1, 2, 2),
+						 X(mktensor_1d)(1, 0, 0),
+						 buf, buf + 1, buf, buf + 1));
+     if (!cldr) goto nada;
+
+     X(ifree)(buf);
+
+     pln->cldr = cldr;
+     pln->cldr0 = cldr0;
+     pln->omega = 0;
+     pln->r = r;
+     pln->g = X(find_generator)(r);
+     pln->ginv = X(power_mod)(pln->g, r - 2, r);
+     pln->kind = kind;
+     A(MULMOD(pln->g, pln->ginv, r) == 1);
+
+     X(ops_add)(&cldr->ops, &cldr->ops, &pln->super.super.ops);
+     pln->super.super.ops.other += (r - 1) * (4 * 2 + 6) + 6;
+     pln->super.super.ops.add += 2 * (r - 1) * 2 + 4;
+     pln->super.super.ops.mul += 2 * (r - 1) * 4;
+
+     return 1;
+
+ nada:
+     X(ifree0)(buf);
+     X(plan_destroy_internal)(cldr);
+     X(plan_destroy_internal)(cldr0);
+     return 0;
+}
+
+static plan *mkplan_dit(const solver *ego, const problem *p_, planner *plnr)
+{
+     const problem_rdft *p = (const problem_rdft *) p_;
+     P *pln = 0;
+     int n, is, os, r, m;
+     plan *cld = (plan *) 0;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego, p_, plnr))
+          goto nada;
+
+     n = p->sz->dims[0].n;
+     is = p->sz->dims[0].is;
+     os = p->sz->dims[0].os;
+
+     r = X(first_divisor)(n);
+     m = n / r;
+
+
+     cld = X(mkplan_d)(plnr, 
+		       X(mkproblem_rdft_d)(X(mktensor_1d)(m, r * is, os),
+					   X(mktensor_1d)(r, is, m * os),
+					   p->I, p->O, p->kind));
+     if (!cld) goto nada;
+
+     pln = MKPLAN_RDFT(P, &padt, apply_dit);
+     if (!mkP(pln, r, p->O, os*m, p->kind[0], plnr))
+	  goto nada;
+
+     pln->ios = os*m;
+     pln->os = os;
+     pln->m = m;
+     pln->cld = cld;
+     pln->W = 0;
+
+     X(ops_madd)((m - 1)/2, &pln->super.super.ops, &cld->ops,
+		 &pln->super.super.ops);
+
+     return &(pln->super.super);
+
+ nada:
+     X(plan_destroy_internal)(cld);
+     X(ifree0)(pln);
+     return (plan *) 0;
+}
+
+static plan *mkplan_dif(const solver *ego, const problem *p_, planner *plnr)
+{
+     const problem_rdft *p = (const problem_rdft *) p_;
+     P *pln = 0;
+     int n, is, os, r, m;
+     plan *cld = (plan *) 0;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego, p_, plnr))
+          goto nada;
+
+     n = p->sz->dims[0].n;
+     is = p->sz->dims[0].is;
+     os = p->sz->dims[0].os;
+
+     r = X(first_divisor)(n);
+     m = n / r;
+
+     cld = X(mkplan_d)(plnr,
+		       X(mkproblem_rdft_d)(X(mktensor_1d)(m, is, r * os),
+					   X(mktensor_1d)(r, m * is, os),
+					   p->I, p->O, p->kind));
+     if (!cld) goto nada;
+
+     pln = MKPLAN_RDFT(P, &padt, apply_dif);
+     if (!mkP(pln, r, p->I, is*m, p->kind[0], plnr)) goto nada;
+
+     pln->ios = is*m;
+     pln->os = is;
+     pln->m = m;
+     pln->cld = cld;
+     pln->W = 0;
+
+     X(ops_madd)((m - 1)/2, &pln->super.super.ops, &cld->ops,
+		 &pln->super.super.ops);
+
+     return &(pln->super.super);
+
+ nada:
+     X(plan_destroy_internal)(cld);
+     X(ifree0)(pln);
+     return (plan *) 0;
+}
+
+/* constructors */
+
+static solver *mksolver_dit(void)
+{
+     static const solver_adt sadt = { mkplan_dit };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->kind = R2HC;
+     return &(slv->super);
+}
+
+static solver *mksolver_dif(void)
+{
+     static const solver_adt sadt = { mkplan_dif };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->kind = HC2R;
+     return &(slv->super);
+}
+
+void X(rdft_rader_hc2hc_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver_dit());
+     REGISTER_SOLVER(p, mksolver_dif());
+}
diff --git a/src/fftw3/rdft/rank-geq2-rdft2.c b/src/fftw3/rdft/rank-geq2-rdft2.c
new file mode 100644
index 0000000..c1809c9
--- /dev/null
+++ b/src/fftw3/rdft/rank-geq2-rdft2.c
@@ -0,0 +1,243 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rank-geq2-rdft2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* plans for RDFT2 of rank >= 2 (multidimensional) */
+
+#include "rdft.h"
+#include "dft.h"
+
+typedef struct {
+     solver super;
+     int spltrnk;
+     const int *buddies;
+     int nbuddies;
+} S;
+
+typedef struct {
+     plan_dft super;
+     plan *cldr, *cldc;
+     const S *solver;
+} P;
+
+static void apply_r2hc(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+
+     {
+	  plan_rdft2 *cldr = (plan_rdft2 *) ego->cldr;
+	  cldr->apply((plan *) cldr, r, rio, iio);
+     }
+     
+     {
+	  plan_dft *cldc = (plan_dft *) ego->cldc;
+	  cldc->apply((plan *) cldc, rio, iio, rio, iio);
+     }
+}
+
+static void apply_hc2r(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+
+     {
+	  plan_dft *cldc = (plan_dft *) ego->cldc;
+	  cldc->apply((plan *) cldc, iio, rio, iio, rio);
+     }
+
+     {
+	  plan_rdft2 *cldr = (plan_rdft2 *) ego->cldr;
+	  cldr->apply((plan *) cldr, r, rio, iio);
+     }
+     
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cldr, flg);
+     AWAKE(ego->cldc, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cldr);
+     X(plan_destroy_internal)(ego->cldc);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     const S *s = ego->solver;
+     p->print(p, "(rdft2-rank>=2/%d%(%p%)%(%p%))", 
+	      s->spltrnk, ego->cldr, ego->cldc);
+}
+ 
+static int picksplit(const S *ego, const tensor *sz, int *rp)
+{
+     A(sz->rnk > 1); /* cannot split rnk <= 1 */
+     if (!X(pickdim)(ego->spltrnk, ego->buddies, ego->nbuddies, sz, 1, rp))
+          return 0;
+     *rp += 1; /* convert from dim. index to rank */
+     if (*rp >= sz->rnk) /* split must reduce rank */
+          return 0;
+     return 1;
+}
+
+static int applicable0(const solver *ego_, const problem *p_, int *rp,
+		       const planner *plnr)
+{
+     if (RDFT2P(p_)) {
+          const problem_rdft2 *p = (const problem_rdft2 *) p_;
+	  const S *ego = (const S *)ego_;
+          return (1
+                  && p->sz->rnk >= 2
+		  && picksplit(ego, p->sz, rp)
+                  && (0
+
+		      /* can work out-of-place, but HC2R destroys input */
+                      || (p->r != p->rio && p->r != p->iio && 
+			  (p->kind == R2HC || DESTROY_INPUTP(plnr)))
+
+		      /* FIXME: what are sufficient conditions for inplace? */
+                      || (!(p->r != p->rio && p->r != p->iio))
+		       )
+	       );
+     }
+
+     return 0;
+}
+
+/* TODO: revise this. */
+static int applicable(const solver *ego_, const problem *p_, 
+		      const planner *plnr, int *rp)
+{
+     const S *ego = (const S *)ego_;
+
+     if (!applicable0(ego_, p_, rp, plnr)) return 0;
+
+     /* fixed spltrnk (unlike fftw2's spltrnk=1, default buddies[0] is
+        spltrnk=0, which is an asymptotic "theoretical optimum" for
+        an ideal cache; it's equivalent to spltrnk=1 for rnk < 4). */
+     if (NO_RANK_SPLITSP(plnr) && (ego->spltrnk != ego->buddies[0]))
+          return 0;
+
+     if (NO_UGLYP(plnr)) {
+	  const problem_rdft2 *p = (const problem_rdft2 *) p_;
+
+	  /* Heuristic: if the vector stride is greater than the transform
+	     sz, don't use (prefer to do the vector loop first with a
+	     vrank-geq1 plan). */
+	  if (p->vecsz->rnk > 0 &&
+	      X(tensor_min_stride)(p->vecsz) 
+	      > X(rdft2_tensor_max_index)(p->sz, p->kind))
+	       return 0;
+     }
+
+     return 1;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     const problem_rdft2 *p;
+     P *pln;
+     plan *cldr = 0, *cldc = 0;
+     tensor *sz1, *sz2, *vecszi, *sz2i;
+     int spltrnk;
+     inplace_kind k;
+     problem *cldp;
+
+     static const plan_adt padt = {
+	  X(rdft2_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr, &spltrnk))
+          return (plan *) 0;
+
+     p = (const problem_rdft2 *) p_;
+     X(tensor_split)(p->sz, &sz1, spltrnk, &sz2);
+
+     k = p->kind == R2HC ? INPLACE_OS : INPLACE_IS;
+     vecszi = X(tensor_copy_inplace)(p->vecsz, k);
+     sz2i = X(tensor_copy_inplace)(sz2, k);
+
+     /* complex data is ~half of real */
+     sz2i->dims[sz2i->rnk - 1].n = sz2i->dims[sz2i->rnk - 1].n/2 + 1;
+
+     cldr = X(mkplan_d)(plnr, 
+		       X(mkproblem_rdft2_d)(X(tensor_copy)(sz2),
+					    X(tensor_append)(p->vecsz, sz1),
+					    p->r, p->rio, p->iio, p->kind));
+     if (!cldr) goto nada;
+
+     if (p->kind == R2HC)
+	  cldp = X(mkproblem_dft_d)(X(tensor_copy_inplace)(sz1, k),
+				    X(tensor_append)(vecszi, sz2i),
+				    p->rio, p->iio, p->rio, p->iio);
+     else /* HC2R must swap re/im parts to get IDFT */
+	  cldp = X(mkproblem_dft_d)(X(tensor_copy_inplace)(sz1, k),
+				    X(tensor_append)(vecszi, sz2i),
+				    p->iio, p->rio, p->iio, p->rio);
+     cldc = X(mkplan_d)(plnr, cldp);
+     if (!cldc) goto nada;
+
+     pln = MKPLAN_RDFT2(P, &padt, p->kind == R2HC ? apply_r2hc : apply_hc2r);
+
+     pln->cldr = cldr;
+     pln->cldc = cldc;
+
+     pln->solver = ego;
+     X(ops_add)(&cldr->ops, &cldc->ops, &pln->super.super.ops);
+
+     X(tensor_destroy4)(sz2i, vecszi, sz2, sz1);
+
+     return &(pln->super.super);
+
+ nada:
+     X(plan_destroy_internal)(cldr);
+     X(plan_destroy_internal)(cldc);
+     X(tensor_destroy4)(sz2i, vecszi, sz2, sz1);
+     return (plan *) 0;
+}
+
+static solver *mksolver(int spltrnk, const int *buddies, int nbuddies)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->spltrnk = spltrnk;
+     slv->buddies = buddies;
+     slv->nbuddies = nbuddies;
+     return &(slv->super);
+}
+
+void X(rdft2_rank_geq2_register)(planner *p)
+{
+     int i;
+     static const int buddies[] = { 0, 1, -2 };
+
+     const int nbuddies = sizeof(buddies) / sizeof(buddies[0]);
+
+     for (i = 0; i < nbuddies; ++i)
+          REGISTER_SOLVER(p, mksolver(buddies[i], buddies, nbuddies));
+
+     /* FIXME: Should we try more buddies?  See also dft/rank-geq2. */
+}
diff --git a/src/fftw3/rdft/rank0-rdft2.c b/src/fftw3/rdft/rank0-rdft2.c
new file mode 100644
index 0000000..5a2a649
--- /dev/null
+++ b/src/fftw3/rdft/rank0-rdft2.c
@@ -0,0 +1,195 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rank0-rdft2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* plans for rank-0 RDFT2 (copy operations, plus setting 0 imag. parts) */
+
+#include "rdft.h"
+
+#ifdef HAVE_STRING_H
+#include <string.h>		/* for memcpy() */
+#endif
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     int vl;
+     int ivs, ovs;
+     plan *cldcpy;
+} P;
+
+static int applicable(const problem *p_)
+{
+     if (RDFT2P(p_)) {
+          const problem_rdft2 *p = (const problem_rdft2 *) p_;
+          return (1
+                  && p->sz->rnk == 0
+		  && (p->kind == HC2R
+		      || (((p->r != p->rio && p->r != p->iio)
+			   || X(rdft2_inplace_strides)(p, RNK_MINFTY))
+			  && p->vecsz->rnk <= 1))
+	       );
+     }
+     return 0;
+}
+
+static void apply_r2hc(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+     int i, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+
+     for (i = 4; i <= vl; i += 4) {
+          R r0, r1, r2, r3;
+          r0 = *r; r += ivs;
+          r1 = *r; r += ivs;
+          r2 = *r; r += ivs;
+          r3 = *r; r += ivs;
+          *rio = r0; rio += ovs;
+	  *iio = 0.0; iio += ovs;
+          *rio = r1; rio += ovs;
+	  *iio = 0.0; iio += ovs;
+          *rio = r2; rio += ovs;
+	  *iio = 0.0; iio += ovs;
+	  *rio = r3; rio += ovs;
+	  *iio = 0.0; iio += ovs;
+     }
+     for (; i < vl + 4; ++i) {
+          R r0;
+          r0 = *r; r += ivs;
+          *rio = r0; rio += ovs;
+	  *iio = 0.0; iio += ovs;
+     }
+}
+
+/* in-place r2hc rank-0: set imaginary parts of output to 0 */
+static void apply_r2hc_inplace(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+     int i, vl = ego->vl;
+     int ovs = ego->ovs;
+
+     UNUSED(r);
+     UNUSED(rio);
+     for (i = 4; i <= vl; i += 4) {
+	  *iio = 0.0; iio += ovs;
+	  *iio = 0.0; iio += ovs;
+	  *iio = 0.0; iio += ovs;
+	  *iio = 0.0; iio += ovs;
+     }
+     for (; i < vl + 4; ++i) {
+	  *iio = 0.0; iio += ovs;
+     }
+}
+
+/* a rank-0 HC2R rdft2 problem is just a copy from rio to r,
+   so we can use a rank-0 rdft plan */
+static void apply_hc2r(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+     plan_rdft *cldcpy = (plan_rdft *) ego->cldcpy;
+     UNUSED(iio);
+     cldcpy->apply((plan *) cldcpy, rio, r);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     if (ego->cldcpy)
+	  AWAKE(ego->cldcpy, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     if (ego->cldcpy)
+	  X(plan_destroy_internal)(ego->cldcpy);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     if (ego->cldcpy)
+	  p->print(p, "(rdft2-hc2r-rank0%(%p%))", ego->cldcpy);
+     else
+	  p->print(p, "(rdft2-r2hc-rank0%v)", ego->vl);
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const problem_rdft2 *p;
+     plan *cldcpy = (plan *) 0;
+     P *pln;
+
+     static const plan_adt padt = {
+	  X(rdft2_solve), awake, print, destroy
+     };
+
+     UNUSED(ego_);
+
+     if (!applicable(p_))
+          return (plan *) 0;
+
+     p = (const problem_rdft2 *) p_;
+
+     if (p->kind == HC2R) {
+	  cldcpy = X(mkplan_d)(plnr,
+			       X(mkproblem_rdft_d)(
+				    X(mktensor_0d)(),
+				    X(tensor_copy)(p->vecsz),
+				    p->rio, p->r, (rdft_kind *) 0));
+	  if (!cldcpy) return (plan *) 0;
+     }
+
+     pln = MKPLAN_RDFT2(P, &padt, 
+			p->kind == R2HC ? 
+			(p->r == p->rio ? apply_r2hc_inplace : apply_r2hc) 
+			: apply_hc2r);
+     
+     if (p->kind == R2HC)
+	  X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     pln->cldcpy = cldcpy;
+
+     if (p->kind == R2HC) {
+	  /* vl loads, 2*vl stores */
+	  X(ops_other)(3 * pln->vl, &pln->super.super.ops);
+     }
+     else {
+	  pln->super.super.ops = cldcpy->ops;
+     }
+
+     return &(pln->super.super);
+}
+
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(rdft2_rank0_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/rdft/rbuffered.c b/src/fftw3/rdft/rbuffered.c
new file mode 100644
index 0000000..19ccc7b
--- /dev/null
+++ b/src/fftw3/rdft/rbuffered.c
@@ -0,0 +1,299 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rbuffered.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+typedef struct {
+     int nbuf;
+     int maxbufsz;
+     int skew_alignment;
+     int skew;
+     const char *nam;
+} bufadt;
+
+typedef struct {
+     solver super;
+     const bufadt *adt;
+} S;
+
+typedef struct {
+     plan_rdft super;
+
+     plan *cld, *cldcpy, *cldrest;
+     int n, vl, nbuf, bufdist;
+     int ivs, ovs;
+
+     const S *slv;
+} P;
+
+/* transform a vector input with the help of bufs */
+static void apply(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     plan_rdft *cld = (plan_rdft *) ego->cld;
+     plan_rdft *cldcpy = (plan_rdft *) ego->cldcpy;
+     plan_rdft *cldrest;
+     int i, vl = ego->vl, nbuf = ego->nbuf;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *bufs;
+
+     bufs = (R *)MALLOC(sizeof(R) * nbuf * ego->bufdist, BUFFERS);
+
+     for (i = nbuf; i <= vl; i += nbuf) {
+          /* transform to bufs: */
+          cld->apply((plan *) cld, I, bufs);
+	  I += ivs;
+
+          /* copy back */
+          cldcpy->apply((plan *) cldcpy, bufs, O);
+	  O += ovs;
+     }
+
+     /* Do the remaining transforms, if any: */
+     cldrest = (plan_rdft *) ego->cldrest;
+     cldrest->apply((plan *) cldrest, I, O);
+
+     X(ifree)(bufs);
+}
+
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+
+     AWAKE(ego->cld, flg);
+     AWAKE(ego->cldcpy, flg);
+     AWAKE(ego->cldrest, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cldrest);
+     X(plan_destroy_internal)(ego->cldcpy);
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(%s-%d%v/%d-%d%(%p%)%(%p%)%(%p%))",
+              ego->slv->adt->nam,
+              ego->n, ego->nbuf,
+              ego->vl, ego->bufdist % ego->n,
+              ego->cld, ego->cldcpy, ego->cldrest);
+}
+
+
+static int compute_nbuf(int n, int vl, const S *ego)
+{
+     return X(compute_nbuf)(n, vl, ego->adt->nbuf, ego->adt->maxbufsz);
+}
+
+static int toobig(int n, const S *ego)
+{
+     return (n > ego->adt->maxbufsz);
+}
+
+static int applicable0(const problem *p_, const S *ego, const planner *plnr)
+{
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          iodim *d = p->sz->dims;
+
+          if (1
+	      && p->vecsz->rnk <= 1
+	      && p->sz->rnk == 1
+	       ) {
+
+	       if (toobig(p->sz->dims[0].n, ego) && CONSERVE_MEMORYP(plnr))
+		    return 0;
+
+               /*
+		 In principle, the buffered transforms might be useful
+		 when working out of place.  However, in order to
+		 prevent infinite loops in the planner, we require
+		 that the output stride of the buffered transforms be
+		 greater than 1.
+               */
+               if (p->I != p->O)
+                    return (d[0].os > 1);
+
+               /* We can always do a single transform in-place */
+               if (p->vecsz->rnk == 0)
+                    return 1;
+
+               /*
+		* If the problem is in place, the input/output strides must
+		* be the same or the whole thing must fit in the buffer.
+		*/
+               return ((X(tensor_inplace_strides2)(p->sz, p->vecsz))
+                       || (compute_nbuf(d[0].n, p->vecsz->dims[0].n, ego)
+                           == p->vecsz->dims[0].n));
+          }
+     }
+     return 0;
+}
+
+static int applicable(const problem *p_, const S *ego, const planner *plnr)
+{
+     const problem_rdft *p;
+
+     if (NO_BUFFERINGP(plnr)) return 0;
+     if (!applicable0(p_, ego, plnr)) return 0;
+
+     p = (const problem_rdft *) p_;
+     if (NO_UGLYP(plnr)) {
+	  if (p->I != p->O) return 0;
+	  if (toobig(p->sz->dims[0].n, ego)) return 0;
+     }
+     return 1;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     const bufadt *adt = ego->adt;
+     P *pln;
+     plan *cld = (plan *) 0;
+     plan *cldcpy = (plan *) 0;
+     plan *cldrest = (plan *) 0;
+     const problem_rdft *p = (const problem_rdft *) p_;
+     R *bufs = (R *) 0;
+     int nbuf = 0, bufdist, n, vl;
+     int ivs, ovs;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+
+     if (!applicable(p_, ego, plnr))
+          goto nada;
+
+     n = X(tensor_sz)(p->sz);
+     X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs);
+
+     nbuf = compute_nbuf(n, vl, ego);
+     A(nbuf > 0);
+
+     /*
+      * Determine BUFDIST, the offset between successive array bufs.
+      * bufdist = n + skew, where skew is chosen such that bufdist %
+      * skew_alignment = skew.
+      */
+     if (vl == 1) {
+          bufdist = n;
+     } else {
+          bufdist =
+               n + ((adt->skew_alignment + adt->skew - n % adt->skew_alignment)
+                    % adt->skew_alignment);
+          A(p->vecsz->rnk == 1);
+     }
+
+     /* initial allocation for the purpose of planning */
+     bufs = (R *) MALLOC(sizeof(R) * nbuf * bufdist, BUFFERS);
+
+     cld = X(mkplan_d)(plnr, 
+		       X(mkproblem_rdft_d)(
+			    X(mktensor_1d)(n, p->sz->dims[0].is, 1),
+			    X(mktensor_1d)(nbuf, ivs, bufdist),
+			    TAINT(p->I, ivs * nbuf), bufs, p->kind));
+     if (!cld) goto nada;
+
+     /* copying back from the buffer is a rank-0 transform: */
+     cldcpy = X(mkplan_d)(plnr, 
+			  X(mkproblem_rdft_d)(
+			       X(mktensor_0d)(),
+			       X(mktensor_2d)(nbuf, bufdist, ovs,
+					      n, 1, p->sz->dims[0].os),
+			       bufs, TAINT(p->O, ovs * nbuf), 
+			       (rdft_kind *) 0));
+     if (!cldcpy) goto nada;
+
+     /* deallocate buffers, let apply() allocate them for real */
+     X(ifree)(bufs);
+     bufs = 0;
+
+     /* plan the leftover transforms (cldrest): */
+     {
+	  int id = ivs * (nbuf * (vl / nbuf));
+	  int od = ovs * (nbuf * (vl / nbuf));
+	  cldrest = X(mkplan_d)(plnr, 
+				X(mkproblem_rdft_d)(
+				     X(tensor_copy)(p->sz),
+				     X(mktensor_1d)(vl % nbuf, ivs, ovs),
+				     p->I + id, p->O + od, p->kind));
+     }
+     if (!cldrest) goto nada;
+
+     pln = MKPLAN_RDFT(P, &padt, apply);
+     pln->cld = cld;
+     pln->cldcpy = cldcpy;
+     pln->cldrest = cldrest;
+     pln->slv = ego;
+     pln->n = n;
+     pln->vl = vl;
+     pln->ivs = ivs * nbuf;
+     pln->ovs = ovs * nbuf;
+
+     pln->nbuf = nbuf;
+     pln->bufdist = bufdist;
+
+     {
+	  opcnt t;
+	  X(ops_add)(&cld->ops, &cldcpy->ops, &t);
+	  X(ops_madd)(vl / nbuf, &t, &cldrest->ops, &pln->super.super.ops);
+     }
+
+     return &(pln->super.super);
+
+ nada:
+     X(ifree0)(bufs);
+     X(plan_destroy_internal)(cldrest);
+     X(plan_destroy_internal)(cldcpy);
+     X(plan_destroy_internal)(cld);
+     return (plan *) 0;
+}
+
+static solver *mksolver(const bufadt *adt)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->adt = adt;
+     return &(slv->super);
+}
+
+
+void X(rdft_buffered_register)(planner *p)
+{
+     /* FIXME: what are good defaults? */
+     static const bufadt adt = {
+	  /* nbuf */           8,
+	  /* maxbufsz */       (65536 / sizeof(R)),
+	  /* skew_alignment */ 8,
+	  /* skew */           5,
+	  /* nam */            "rdft-buffered"
+     };
+
+     REGISTER_SOLVER(p, mksolver(&adt));
+}
diff --git a/src/fftw3/rdft/rconf.c b/src/fftw3/rdft/rconf.c
new file mode 100644
index 0000000..2c65d4c
--- /dev/null
+++ b/src/fftw3/rdft/rconf.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rconf.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+static const solvtab s =
+{
+     SOLVTAB(X(rdft_indirect_register)),
+     SOLVTAB(X(rdft_rank0_register)),
+     SOLVTAB(X(rdft_vrank_geq1_register)),
+     /* 
+	SOLVTAB(X(rdft_vrank2_transpose_register)),
+	SOLVTAB(X(rdft_vrank3_transpose_register)),
+     */
+     SOLVTAB(X(rdft_nop_register)),
+     SOLVTAB(X(rdft_buffered_register)),
+     SOLVTAB(X(rdft_rader_hc2hc_register)),
+     SOLVTAB(X(rdft_generic_register)),
+     SOLVTAB(X(rdft_rank_geq2_register)),
+
+     SOLVTAB(X(dft_r2hc_register)),
+
+     SOLVTAB(X(rdft_dht_register)),
+     SOLVTAB(X(dht_r2hc_register)),
+     SOLVTAB(X(dht_rader_register)),
+
+     SOLVTAB(X(rdft2_vrank_geq1_register)),
+     SOLVTAB(X(rdft2_nop_register)),
+     SOLVTAB(X(rdft2_rank0_register)),
+     SOLVTAB(X(rdft2_buffered_register)),
+     SOLVTAB(X(rdft2_rank_geq2_register)),
+     SOLVTAB(X(rdft2_radix2_register)),
+
+     SOLVTAB_END
+};
+
+void X(rdft_conf_standard)(planner *p)
+{
+     X(solvtab_exec)(s, p);
+     X(solvtab_exec)(X(solvtab_rdft_r2hc), p);
+     X(solvtab_exec)(X(solvtab_rdft_hc2r), p);
+     X(solvtab_exec)(X(solvtab_rdft_r2r), p);
+}
diff --git a/src/fftw3/rdft/rdft-dht.c b/src/fftw3/rdft/rdft-dht.c
new file mode 100644
index 0000000..f384fef
--- /dev/null
+++ b/src/fftw3/rdft/rdft-dht.c
@@ -0,0 +1,223 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rdft-dht.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* Solve an R2HC/HC2R problem via post/pre processing of a DHT.  This
+   is mainly useful because we can use Rader to compute DHTs of prime
+   sizes.  It also allows us to express hc2r problems in terms of r2hc
+   (via dht-r2hc), and to do hc2r problems without destroying the input. */
+
+#include "rdft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cld;
+     int is, os;
+     int n;
+} P;
+
+static void apply_r2hc(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int os;
+     int i, n;
+
+     {
+	  plan_rdft *cld = (plan_rdft *) ego->cld;
+	  cld->apply((plan *) cld, I, O);
+     }
+
+     n = ego->n;
+     os = ego->os;
+     for (i = 1; i < n - i; ++i) {
+	  E a, b;
+	  a = K(0.5) * O[os * i];
+	  b = K(0.5) * O[os * (n - i)];
+	  O[os * i] = a + b;
+#if FFT_SIGN == -1
+	  O[os * (n - i)] = b - a;
+#else
+	  O[os * (n - i)] = a - b;
+#endif
+     }
+}
+
+/* hc2r, destroying input as usual */
+static void apply_hc2r(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is;
+     int i, n = ego->n;
+
+     for (i = 1; i < n - i; ++i) {
+	  E a, b;
+	  a = I[is * i];
+	  b = I[is * (n - i)];
+#if FFT_SIGN == -1
+	  I[is * i] = a - b;
+	  I[is * (n - i)] = a + b;
+#else
+	  I[is * i] = a + b;
+	  I[is * (n - i)] = a - b;
+#endif
+     }
+
+     {
+	  plan_rdft *cld = (plan_rdft *) ego->cld;
+	  cld->apply((plan *) cld, I, O);
+     }
+}
+
+/* hc2r, without destroying input */
+static void apply_hc2r_save(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n;
+
+     O[0] = I[0];
+     for (i = 1; i < n - i; ++i) {
+	  E a, b;
+	  a = I[is * i];
+	  b = I[is * (n - i)];
+#if FFT_SIGN == -1
+	  O[os * i] = a - b;
+	  O[os * (n - i)] = a + b;
+#else
+	  O[os * i] = a + b;
+	  O[os * (n - i)] = a - b;
+#endif
+     }
+     if (i == n - i)
+	  O[os * i] = I[is * i];
+
+     {
+	  plan_rdft *cld = (plan_rdft *) ego->cld;
+	  cld->apply((plan *) cld, O, O);
+     }
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cld, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(%s-dht-%d%(%p%))", 
+	      ego->super.apply == apply_r2hc ? "r2hc" : "hc2r",
+	      ego->n, ego->cld);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk == 0
+		  && (p->kind[0] == R2HC || p->kind[0] == HC2R)
+
+		  /* hack: size-2 DHT etc. are defined as being equivalent
+		     to size-2 R2HC in problem.c, so we need this to prevent
+		     infinite loops for size 2 in EXHAUSTIVE mode: */
+		  && p->sz->dims[0].n > 2
+	       );
+     }
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p_, 
+		      const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0(ego, p_));
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     P *pln;
+     const problem_rdft *p;
+     problem *cldp;
+     plan *cld;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+          return (plan *)0;
+
+     p = (const problem_rdft *) p_;
+
+     if (p->kind[0] == R2HC || DESTROY_INPUTP(plnr))
+	  cldp = X(mkproblem_rdft_1)(p->sz, p->vecsz, p->I, p->O, DHT);
+     else {
+	  tensor *sz = X(tensor_copy_inplace)(p->sz, INPLACE_OS);
+	  cldp = X(mkproblem_rdft_1)(sz, p->vecsz, p->O, p->O, DHT);
+	  X(tensor_destroy)(sz);
+     }
+     cld = X(mkplan_d)(plnr, cldp);
+     if (!cld) return (plan *)0;
+
+     pln = MKPLAN_RDFT(P, &padt, p->kind[0] == R2HC ? 
+		       apply_r2hc : (DESTROY_INPUTP(plnr) ?
+				     apply_hc2r : apply_hc2r_save));
+     pln->n = p->sz->dims[0].n;
+     pln->is = p->sz->dims[0].is;
+     pln->os = p->sz->dims[0].os;
+     pln->cld = cld;
+     
+     pln->super.super.ops = cld->ops;
+     pln->super.super.ops.other += 4 * ((pln->n - 1)/2);
+     pln->super.super.ops.add += 2 * ((pln->n - 1)/2);
+     if (p->kind[0] == R2HC)
+	  pln->super.super.ops.mul += 2 * ((pln->n - 1)/2);
+     if (pln->super.apply == apply_hc2r_save)
+	  pln->super.super.ops.other += 2 + (pln->n % 2 ? 0 : 2);
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(rdft_dht_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/rdft/rdft.h b/src/fftw3/rdft/rdft.h
new file mode 100644
index 0000000..375b278
--- /dev/null
+++ b/src/fftw3/rdft/rdft.h
@@ -0,0 +1,161 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#ifndef __RDFT_H__
+#define __RDFT_H__
+
+#include "ifftw.h"
+#include "codelet-rdft.h"
+
+/* problem.c: */
+typedef struct {
+     problem super;
+     tensor *sz, *vecsz;
+     R *I, *O;
+#if defined(STRUCT_HACK_KR)
+     rdft_kind kind[1];
+#elif defined(STRUCT_HACK_C99)
+     rdft_kind kind[];
+#else
+     rdft_kind *kind;
+#endif
+} problem_rdft;
+
+int X(problem_rdft_p)(const problem *p);
+#define RDFTP X(problem_rdft_p)  /* shorthand */
+
+void X(rdft_zerotens)(tensor *sz, R *I);
+problem *X(mkproblem_rdft)(const tensor *sz, const tensor *vecsz,
+			   R *I, R *O, const rdft_kind *kind);
+problem *X(mkproblem_rdft_d)(tensor *sz, tensor *vecsz,
+			     R *I, R *O, const rdft_kind *kind);
+problem *X(mkproblem_rdft_1)(const tensor *sz, const tensor *vecsz,
+			     R *I, R *O, rdft_kind kind);
+problem *X(mkproblem_rdft_1_d)(tensor *sz, tensor *vecsz,
+			       R *I, R *O, rdft_kind kind);
+
+const char *X(rdft_kind_str)(rdft_kind kind);
+
+/* solve.c: */
+void X(rdft_solve)(const plan *ego_, const problem *p_);
+
+/* plan.c: */
+typedef void (*rdftapply) (const plan *ego, R *I, R *O);
+
+typedef struct {
+     plan super;
+     rdftapply apply;
+} plan_rdft;
+
+plan *X(mkplan_rdft)(size_t size, const plan_adt *adt, rdftapply apply);
+
+#define MKPLAN_RDFT(type, adt, apply) \
+  (type *)X(mkplan_rdft)(sizeof(type), adt, apply)
+
+/* various solvers */
+
+solver *X(mksolver_rdft_r2hc_direct)(kr2hc k, const kr2hc_desc *desc);
+solver *X(mksolver_rdft_hc2r_direct)(khc2r k, const khc2r_desc *desc);
+solver *X(mksolver_rdft_r2r_direct)(kr2r k, const kr2r_desc *desc);
+solver *X(mksolver_rdft_hc2hc_dit)(khc2hc k, const hc2hc_desc *desc);
+solver *X(mksolver_rdft_hc2hc_ditbuf)(khc2hc k, const hc2hc_desc *desc);
+solver *X(mksolver_rdft_hc2hc_dif)(khc2hc k, const hc2hc_desc *desc);
+solver *X(mksolver_rdft_hc2hc_difbuf)(khc2hc k, const hc2hc_desc *desc);
+
+extern void (*X(khc2hc_dit_register_hook))(planner *, khc2hc, const hc2hc_desc *);
+extern void (*X(khc2hc_dif_register_hook))(planner *, khc2hc, const hc2hc_desc *);
+
+void X(rdft_rank0_register)(planner *p);
+void X(rdft_rank_geq2_register)(planner *p);
+void X(rdft_indirect_register)(planner *p);
+void X(rdft_vrank_geq1_register)(planner *p);
+void X(rdft_vrank2_transpose_register)(planner *p);
+void X(rdft_vrank3_transpose_register)(planner *p);
+void X(rdft_buffered_register)(planner *p);
+void X(rdft_generic_register)(planner *p);
+void X(rdft_rader_hc2hc_register)(planner *p);
+void X(rdft_dht_register)(planner *p);
+void X(dht_r2hc_register)(planner *p);
+void X(dht_rader_register)(planner *p);
+void X(dft_r2hc_register)(planner *p);
+void X(rdft_nop_register)(planner *p);
+
+/****************************************************************************/
+/* problem2.c: */
+/* an RDFT2 problem transforms a 1d real array r[n] with stride is/os
+   to/from an "unpacked" complex array {rio,iio}[n/2 + 1] with stride
+   os/is.  Multidimensional transforms use complex DFTs for the
+   noncontiguous dimensions.  vecsz has the usual interpretation.  */
+typedef struct {
+     problem super;
+     tensor *sz;
+     tensor *vecsz;
+     R *r, *rio, *iio;
+     rdft_kind kind; /* R2HC or HC2R */
+} problem_rdft2;
+
+int X(problem_rdft2_p)(const problem *p);
+#define RDFT2P X(problem_rdft2_p)  /* shorthand */
+
+problem *X(mkproblem_rdft2)(const tensor *sz, const tensor *vecsz,
+			    R *r, R *rio, R *iio, rdft_kind kind);
+problem *X(mkproblem_rdft2_d)(tensor *sz, tensor *vecsz,
+			      R *r, R *rio, R *iio, rdft_kind kind);
+int X(rdft2_inplace_strides)(const problem_rdft2 *p, int vdim);
+int X(rdft2_tensor_max_index)(const tensor *sz, rdft_kind k);
+void X(rdft2_strides)(rdft_kind kind, const iodim *d, int *is, int *os);
+
+/* verify.c: */
+void X(rdft2_verify)(plan *pln, const problem_rdft2 *p, int rounds);
+
+/* solve.c: */
+void X(rdft2_solve)(const plan *ego_, const problem *p_);
+
+/* plan.c: */
+typedef void (*rdft2apply) (const plan *ego, R *r, R *rio, R *iio);
+
+typedef struct {
+     plan super;
+     rdft2apply apply;
+} plan_rdft2;
+
+plan *X(mkplan_rdft2)(size_t size, const plan_adt *adt, rdft2apply apply);
+
+#define MKPLAN_RDFT2(type, adt, apply) \
+  (type *)X(mkplan_rdft2)(sizeof(type), adt, apply)
+
+/* various solvers */
+
+solver *X(mksolver_rdft2_r2hc_direct)(kr2hc k, const kr2hc_desc *desc);
+solver *X(mksolver_rdft2_hc2r_direct)(khc2r k, const khc2r_desc *desc);
+
+void X(rdft2_vrank_geq1_register)(planner *p);
+void X(rdft2_buffered_register)(planner *p);
+void X(rdft2_nop_register)(planner *p);
+void X(rdft2_rank0_register)(planner *p);
+void X(rdft2_rank_geq2_register)(planner *p);
+void X(rdft2_radix2_register)(planner *p);
+
+/****************************************************************************/
+
+/* configurations */
+void X(rdft_conf_standard)(planner *p);
+
+#endif /* __RDFT_H__ */
diff --git a/src/fftw3/rdft/rdft2-inplace-strides.c b/src/fftw3/rdft/rdft2-inplace-strides.c
new file mode 100644
index 0000000..2171672
--- /dev/null
+++ b/src/fftw3/rdft/rdft2-inplace-strides.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rdft2-inplace-strides.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+/* Check if the vecsz/sz strides are consistent with the problem
+   being in-place for vecsz.dim[vdim], or for all dimensions
+   if vdim == RNK_MINFTY.  We can't just use tensor_inplace_strides
+   because rdft transforms have the unfortunate property of
+   differing input and output sizes.   This routine is not
+   exhaustive; we only return 1 for the most common case.  */
+int X(rdft2_inplace_strides)(const problem_rdft2 *p, int vdim)
+{
+     int N, Nc;
+     int is, os;
+     int i;
+     
+     for (i = 0; i + 1 < p->sz->rnk; ++i)
+	  if (p->sz->dims[i].is != p->sz->dims[i].os)
+	       return 0;
+
+     if (!FINITE_RNK(p->vecsz->rnk) || p->vecsz->rnk == 0)
+	  return 1;
+     if (!FINITE_RNK(vdim)) { /* check all vector dimensions */
+	  for (vdim = 0; vdim < p->vecsz->rnk; ++vdim)
+	       if (!X(rdft2_inplace_strides)(p, vdim))
+		    return 0;
+	  return 1;
+     }
+
+     A(vdim < p->vecsz->rnk);
+     if (p->sz->rnk == 0)
+	  return(p->vecsz->dims[vdim].is == p->vecsz->dims[vdim].os);
+
+     N = X(tensor_sz)(p->sz);
+     Nc = (N / p->sz->dims[p->sz->rnk-1].n) *
+	  (p->sz->dims[p->sz->rnk-1].n/2 + 1);
+     X(rdft2_strides)(p->kind, p->sz->dims + p->sz->rnk - 1, &is, &os);
+     return(p->vecsz->dims[vdim].is == p->vecsz->dims[vdim].os
+	    && X(iabs)(p->vecsz->dims[vdim].os)
+	    >= X(imax)(Nc * X(iabs)(os), N * X(iabs)(is)));
+}
diff --git a/src/fftw3/rdft/rdft2-radix2.c b/src/fftw3/rdft/rdft2-radix2.c
new file mode 100644
index 0000000..280d642
--- /dev/null
+++ b/src/fftw3/rdft/rdft2-radix2.c
@@ -0,0 +1,479 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rdft2-radix2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/*
+  Compute RDFT2 of even size via either a DFT or a vector RDFT of
+  size n/2.
+
+  This file is meant as a temporary hack until we do the right thing.
+
+  The right thing is: 1) get rid of reduction to DFT, and 2) implement
+  arbitrary even-radix reduction to RDFT.  We currently reduce to DFT
+  so as to exploit the SIMD code.  We currently do only radix-2 in
+  order to avoid generating yet another set of codelets.
+*/
+
+#include "rdft.h"
+#include "dft.h"
+
+typedef struct {
+     int (*applicable) (const problem *p_, const planner *plnr);
+     void (*apply) (const plan *ego_, R *r, R *rio, R *iio);
+     problem *(*mkcld) (const problem_rdft2 *p);
+     opcnt ops;
+     const char *nam;
+} madt;
+
+typedef struct {
+     solver super;
+     const madt *adt;
+} S;
+
+typedef struct {
+     plan_dft super;
+     plan *cld;
+     twid *td;
+     int is, os, ivs, ovs;
+     int n, vl;
+     const S *slv;
+} P;
+
+/* common applicability function of forward problems */
+static int applicable_f(const problem *p_, const planner *plnr)
+{
+     UNUSED(plnr);
+     if (RDFT2P(p_)) {
+          const problem_rdft2 *p = (const problem_rdft2 *) p_;
+          return (1
+                  && p->kind == R2HC
+                  && p->vecsz->rnk <= 1
+                  && p->sz->rnk == 1
+		  && (p->sz->dims[0].n % 2) == 0
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable_f_dft(const problem *p_, const planner *plnr)
+{
+     UNUSED(plnr);
+     if (applicable_f(p_, plnr)) {
+	  const problem_rdft2 *p = (const problem_rdft2 *) p_;
+	  return(p->r != p->rio
+		 || (p->iio == p->rio + p->sz->dims[0].is
+		     && p->sz->dims[0].os == 2 * p->sz->dims[0].is));
+     }
+     return 0;
+}
+
+/* common applicability function of backward problems */
+static int applicable_b(const problem *p_, const planner *plnr)
+{
+     if (RDFT2P(p_)) {
+          const problem_rdft2 *p = (const problem_rdft2 *) p_;
+          return (1
+                  && p->kind == HC2R
+		  && (p->r == p->rio || DESTROY_INPUTP(plnr))
+                  && p->vecsz->rnk <= 1
+                  && p->sz->rnk == 1
+		  && (p->sz->dims[0].n % 2) == 0
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable_b_dft(const problem *p_, const planner *plnr)
+{
+     UNUSED(plnr);
+     if (applicable_b(p_, plnr)) {
+	  const problem_rdft2 *p = (const problem_rdft2 *) p_;
+	  return(p->r != p->rio
+		 || (p->iio == p->rio + p->sz->dims[0].os
+		     && p->sz->dims[0].is == 2 * p->sz->dims[0].os));
+     }
+     return 0;
+}
+
+/*
+ * forward rdft2 via dft
+ */
+static void k_f_dft(R *rio, R *iio, const R *W, int n, int dist)
+{
+     int i;
+     R *pp = rio, *pm = rio + n * dist;
+     int im = iio - rio;
+
+     /* i = 0 and i = n */
+     {
+          E rop = pp[0], iop = pp[im];
+          pp[0] = rop + iop;
+          pm[0] = rop - iop;
+          pp[im] = K(0.0);
+          pm[im] = K(0.0);
+	  pp += dist; pm -= dist;
+     }
+
+     /* middle elements */
+     for (W += 2, i = 2; i < n; i += 2, W += 2) {
+          E rop = pp[0], iop = pp[im], rom = pm[0], iom = pm[im];
+          E wr = W[0], wi = W[1];
+          E re = rop + rom;
+          E ie = iop - iom;
+          E rd = rom - rop;
+          E id = iop + iom;
+          E tr = rd * wr - id * wi;
+          E ti = id * wr + rd * wi;
+          pp[0] = K(0.5) * (re + ti);
+          pp[im] = K(0.5) * (ie + tr);
+          pm[0] = K(0.5) * (re - ti);
+          pm[im] = K(0.5) * (tr - ie);
+	  pp += dist; pm -= dist;
+     }
+
+     /* i = n/2 when n is even */
+     if (!(n & 1)) pp[im] = -pp[im];
+}
+
+static void apply_f_dft(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+
+     {
+          /* transform input as a vector of complex numbers */
+          plan_dft *cld = (plan_dft *) ego->cld;
+          cld->apply((plan *) cld, r, r + ego->is, rio, iio);
+     }
+
+     {
+          int i, vl = ego->vl, n2 = ego->n / 2;
+          int ovs = ego->ovs, os = ego->os;
+          const R *W = ego->td->W;
+          for (i = 0; i < vl; ++i, rio += ovs, iio += ovs)
+               k_f_dft(rio, iio, W, n2, os);
+     }
+}
+
+static problem *mkcld_f_dft(const problem_rdft2 *p)
+{
+     const iodim *d = p->sz->dims;
+     return X(mkproblem_dft_d) (
+	  X(mktensor_1d)(d[0].n / 2, d[0].is * 2, d[0].os),
+	  X(tensor_copy)(p->vecsz),
+	  p->r, p->r + d[0].is, p->rio, p->iio);
+}
+
+static const madt adt_f_dft = {
+     applicable_f_dft, apply_f_dft, mkcld_f_dft, {10, 8, 0, 0}, "r2hc2-dft"
+};
+
+/*
+ * forward rdft2 via rdft
+ */
+static void k_f_rdft(R *rio, R *iio, const R *W, int n, int dist)
+{
+     int i;
+     R *pp = rio, *pm = rio + n * dist;
+     int im = iio - rio;
+
+     /* i = 0 and i = n */
+     {
+          E rop = pp[0], iop = pp[im];
+          pp[0] = rop + iop;
+          pm[0] = rop - iop;
+          pp[im] = K(0.0);
+          pm[im] = K(0.0);
+	  pp += dist; pm -= dist;
+     }
+
+     /* middle elements */
+     for (W += 2, i = 2; i < n; i += 2, W += 2) {
+          E r0 = pp[0], r1 = pp[im], i0 = pm[0], i1 = pm[im];
+          E wr = W[0], wi = W[1];
+          E tr = r1 * wr + i1 * wi;
+          E ti = i1 * wr - r1 * wi;
+          pp[0] = r0 + tr;
+          pp[im] = i0 + ti;
+          pm[0] = r0 - tr;
+          pm[im] = ti - i0;
+	  pp += dist; pm -= dist;
+     }
+
+     /* i = n/2 when n is even */
+     if (!(n & 1)) pp[im] = -pp[im];
+}
+
+static void apply_f_rdft(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+
+     {
+          plan_rdft *cld = (plan_rdft *) ego->cld;
+          cld->apply((plan *) cld, r, rio);
+     }
+
+     {
+          int i, vl = ego->vl, n2 = ego->n / 2;
+          int ovs = ego->ovs, os = ego->os;
+          const R *W = ego->td->W;
+          for (i = 0; i < vl; ++i, rio += ovs, iio += ovs)
+               k_f_rdft(rio, iio, W, n2, os);
+     }
+}
+
+static problem *mkcld_f_rdft(const problem_rdft2 *p)
+{
+     const iodim *d = p->sz->dims;
+
+     tensor *radix = X(mktensor_1d)(2, d[0].is, p->iio - p->rio);
+     tensor *cld_vec = X(tensor_append)(radix, p->vecsz);
+     X(tensor_destroy)(radix);
+
+     return X(mkproblem_rdft_1_d) (
+	  X(mktensor_1d)(d[0].n / 2, 2 * d[0].is, d[0].os),
+	  cld_vec, p->r, p->rio, R2HC);
+}
+
+static const madt adt_f_rdft = {
+     applicable_f, apply_f_rdft, mkcld_f_rdft, {6, 4, 0, 0}, "r2hc2-rdft"
+};
+
+
+/*
+ * backward rdft2 via dft
+ */
+static void k_b_dft(R *rio, R *iio, const R *W, int n, int dist)
+{
+     int i;
+     R *pp = rio, *pm = rio + n * dist;
+     int im = iio - rio;
+
+     /* i = 0 and i = n */
+     {
+          E rop = pp[0], iop = pm[0];
+          pp[0] = rop + iop;
+          pp[im] = rop - iop;
+	  pp += dist; pm -= dist;
+     }
+
+     /* middle elements */
+     for (W += 2, i = 2; i < n; i += 2, W += 2) {
+          E a = pp[0], b = pp[im], c = pm[0], d = pm[im];
+          E wr = W[0], wi = W[1];
+	  E re = a + c, ti = a - c, ie = b - d, tr = b + d;
+	  E rd = tr * wr + ti * wi;
+	  E id = ti * wr - tr * wi;
+	  pp[0] = re - rd;
+	  pp[im] = ie + id;
+	  pm[0] = re + rd;
+	  pm[im] = id - ie;
+	  pp += dist; pm -= dist;
+     }
+
+     /* i = n/2 when n is even */
+     if (!(n & 1)) { pp[0] *= K(2.0); pp[im] *= -K(2.0); }
+}
+
+static void apply_b_dft(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+     {
+          int i, vl = ego->vl, n2 = ego->n / 2;
+          int ivs = ego->ivs, is = ego->is;
+          const R *W = ego->td->W;
+	  R *rio1 = rio, *iio1 = iio;
+          for (i = 0; i < vl; ++i, rio1 += ivs, iio1 += ivs)
+               k_b_dft(rio1, iio1, W, n2, is);
+     }
+
+     {
+          plan_dft *cld = (plan_dft *) ego->cld;
+	  /* swap r/i because of backward transform */
+          cld->apply((plan *) cld, iio, rio, r + ego->os, r);
+     }
+}
+
+static problem *mkcld_b_dft(const problem_rdft2 *p)
+{
+     const iodim *d = p->sz->dims;
+
+     return X(mkproblem_dft_d) (
+	  X(mktensor_1d)(d[0].n / 2, d[0].is, 2 * d[0].os),
+	  X(tensor_copy)(p->vecsz),
+	  p->iio, p->rio, p->r + d[0].os, p->r);
+}
+
+static const madt adt_b_dft = {
+     applicable_b_dft, apply_b_dft, mkcld_b_dft, {10, 8, 0, 0}, "hc2r2-dft"
+};
+
+/*
+ * backward rdft2 via backward rdft
+ */
+static void k_b_rdft(R *rio, R *iio, const R *W, int n, int dist)
+{
+     int i;
+     R *pp = rio, *pm = rio + n * dist;
+     int im = iio - rio;
+
+     /* i = 0 and i = n */
+     {
+          E rop = pp[0], iop = pm[0];
+          pp[0] = rop + iop;
+          pp[im] = rop - iop;
+	  pp += dist; pm -= dist;
+     }
+
+     /* middle elements */
+     for (W += 2, i = 2; i < n; i += 2, W += 2) {
+          E a = pp[0], b = pp[im], c = pm[0], d = pm[im];
+          E wr = W[0], wi = W[1];
+	  E r0 = a + c, r1 = a - c, i0 = b - d, i1 = b + d;
+	  pp[0] = r0;
+	  pm[0] = i0;
+	  pp[im] = r1 * wr - i1 * wi;
+	  pm[im] = i1 * wr + r1 * wi;
+	  pp += dist; pm -= dist;
+     }
+
+     /* i = n/2 when n is even */
+     if (!(n & 1)) { pp[0] *= K(2.0); pp[im] *= -K(2.0); }
+}
+
+static void apply_b_rdft(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+
+     {
+          int i, vl = ego->vl, n2 = ego->n / 2;
+          int ivs = ego->ivs, is = ego->is;
+          const R *W = ego->td->W;
+	  R *rio1 = rio, *iio1 = iio;
+          for (i = 0; i < vl; ++i, rio1 += ivs, iio1 += ivs)
+               k_b_rdft(rio1, iio1, W, n2, is);
+     }
+
+     {
+          plan_rdft *cld = (plan_rdft *) ego->cld;
+          cld->apply((plan *) cld, rio, r);
+     }
+}
+
+static problem *mkcld_b_rdft(const problem_rdft2 *p)
+{
+     const iodim *d = p->sz->dims;
+
+     tensor *radix = X(mktensor_1d)(2, p->iio - p->rio, d[0].os);
+     tensor *cld_vec = X(tensor_append)(radix, p->vecsz);
+     X(tensor_destroy)(radix);
+
+     return X(mkproblem_rdft_1_d) (
+	  X(mktensor_1d)(d[0].n / 2, d[0].is, 2 * d[0].os),
+	  cld_vec, p->rio, p->r, HC2R);
+}
+
+static const madt adt_b_rdft = {
+     applicable_b, apply_b_rdft, mkcld_b_rdft, {6, 4, 0, 0}, "hc2r2-rdft"
+};
+
+/*
+ * common stuff
+ */
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     static const tw_instr twinstr[] = { {TW_FULL, 0, 2}, {TW_NEXT, 1, 0} };
+     AWAKE(ego->cld, flg);
+     X(twiddle_awake)(flg, &ego->td, twinstr, ego->n, 2, (ego->n / 2 + 1) / 2);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal) (ego->cld);
+}
+
+static void print(const plan *ego_, printer * p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(%s-%d%v%(%p%))", ego->slv->adt->nam,
+              ego->n, ego->vl, ego->cld);
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     P *pln;
+     const problem_rdft2 *p;
+     plan *cld;
+     const iodim *d;
+
+     static const plan_adt padt = {
+          X(rdft2_solve), awake, print, destroy
+     };
+
+     if (!ego->adt->applicable(p_, plnr))
+          return (plan *) 0;
+
+     p = (const problem_rdft2 *) p_;
+
+     cld = X(mkplan_d)(plnr, ego->adt->mkcld(p));
+     if (!cld) return (plan *) 0;
+
+     pln = MKPLAN_RDFT2(P, &padt, ego->adt->apply);
+
+     d = p->sz->dims;
+     pln->n = d[0].n;
+     pln->os = d[0].os;
+     pln->is = d[0].is;
+     X(tensor_tornk1) (p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     pln->cld = cld;
+     pln->td = 0;
+     pln->slv = ego;
+
+     /* approximately */
+     X(ops_madd)(pln->vl * ((pln->n/2 + 1) / 2), &ego->adt->ops,
+		 &cld->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+}
+
+static solver *mksolver(const madt *adt)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->adt = adt;
+     return &(slv->super);
+}
+
+void X(rdft2_radix2_register)(planner *p)
+{
+     unsigned i;
+     static const madt *const adts[] = {
+	  &adt_f_dft, &adt_f_rdft,
+	  &adt_b_dft, &adt_b_rdft
+     };
+
+     for (i = 0; i < sizeof(adts) / sizeof(adts[0]); ++i)
+          REGISTER_SOLVER(p, mksolver(adts[i]));
+}
diff --git a/src/fftw3/rdft/rdft2-strides.c b/src/fftw3/rdft/rdft2-strides.c
new file mode 100644
index 0000000..bcd98d2
--- /dev/null
+++ b/src/fftw3/rdft/rdft2-strides.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "rdft.h"
+
+/* Deal with annoyance because the tensor (is,os) applies to
+   (r,rio/iio) for R2HC and vice-versa for HC2R.  We originally had
+   (is,os) always apply to (r,rio/iio), but this causes other
+   headaches with the tensor functions. */
+void X(rdft2_strides)(rdft_kind kind, const iodim *d, int *is, int *os)
+{
+     if (kind == R2HC) {
+	  *is = d->is;
+	  *os = d->os;
+     }
+     else {
+	  A(kind == HC2R);
+	  *is = d->os;
+	  *os = d->is;
+     }
+}
diff --git a/src/fftw3/rdft/rdft2-tensor-max-index.c b/src/fftw3/rdft/rdft2-tensor-max-index.c
new file mode 100644
index 0000000..0d29af3
--- /dev/null
+++ b/src/fftw3/rdft/rdft2-tensor-max-index.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rdft2-tensor-max-index.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+/* like X(tensor_max_index), but takes into account the special n/2+1
+   final dimension for the complex output/input of an R2HC/HC2R transform. */
+int X(rdft2_tensor_max_index)(const tensor *sz, rdft_kind k)
+{
+     int i;
+     int n = 0;
+
+     A(FINITE_RNK(sz->rnk));
+     for (i = 0; i + 1 < sz->rnk; ++i) {
+          const iodim *p = sz->dims + i;
+          n += (p->n - 1) * X(imax)(X(iabs)(p->is), X(iabs)(p->os));
+     }
+     if (i < sz->rnk) {
+	  const iodim *p = sz->dims + i;
+	  int is, os;
+	  X(rdft2_strides)(k, p, &is, &os);
+	  n += X(imax)((p->n - 1) * X(iabs)(is), (p->n/2) * X(iabs)(os));
+     }
+     return n;
+}
diff --git a/src/fftw3/rdft/rdirect.c b/src/fftw3/rdft/rdirect.c
new file mode 100644
index 0000000..97dd16e
--- /dev/null
+++ b/src/fftw3/rdft/rdirect.c
@@ -0,0 +1,259 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rdirect.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* direct RDFT R2HC/HC2R solver, if we have a codelet */
+
+#include "rdft.h"
+
+typedef union {
+     kr2hc r2hc;
+     khc2r hc2r;
+     kr2r r2r;
+} kodelet;
+
+typedef struct {
+     solver super;
+     union {
+	  const kr2hc_desc *r2hc;
+	  const khc2r_desc *hc2r;
+	  const kr2r_desc *r2r;
+     } desc;
+     kodelet k;
+     int sz;
+     rdft_kind kind;
+     const char *nam;
+} S;
+
+typedef struct {
+     plan_rdft super;
+
+     stride is, ros, ios;
+     int ioffset;
+     int vl;
+     int ivs, ovs;
+     kodelet k;
+     const S *slv;
+} P;
+
+static void apply_r2hc(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     ASSERT_ALIGNED_DOUBLE;
+     ego->k.r2hc(I, O, O + ego->ioffset, ego->is, ego->ros, ego->ios,
+		 ego->vl, ego->ivs, ego->ovs);
+}
+
+static void apply_hc2r(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     ASSERT_ALIGNED_DOUBLE;
+     ego->k.hc2r(I, I + ego->ioffset, O, ego->ros, ego->ios, ego->is,
+		 ego->vl, ego->ivs, ego->ovs);
+}
+
+static void apply_r2r(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     ASSERT_ALIGNED_DOUBLE;
+     ego->k.r2r(I, O, ego->is, ego->ros, ego->vl, ego->ivs, ego->ovs);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(stride_destroy)(ego->is);
+     X(stride_destroy)(ego->ros);
+     if (!R2R_KINDP(ego->slv->kind))
+	  X(stride_destroy)(ego->ios);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     const S *s = ego->slv;
+
+     p->print(p, "(rdft-%s-direct-%d%v \"%s\")", 
+	      X(rdft_kind_str)(s->kind), s->sz, ego->vl, s->nam);
+}
+
+static int ioffset(rdft_kind kind, int sz, int s)
+{
+     return(s * ((kind == R2HC || kind == HC2R) ? sz : (sz - 1)));
+}
+
+static int applicable(const solver *ego_, const problem *p_)
+{
+     if (RDFTP(p_)) {
+          const S *ego = (const S *) ego_;
+          const problem_rdft *p = (const problem_rdft *) p_;
+	  int vl;
+	  int ivs, ovs;
+
+          return (
+	       1
+	       && p->sz->rnk == 1
+	       && p->vecsz->rnk <= 1
+	       && p->sz->dims[0].n == ego->sz
+	       && p->kind[0] == ego->kind
+
+	       /* check strides etc */
+	       && X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs)
+
+	       && (!R2HC_KINDP(ego->kind) ||
+		   ego->desc.r2hc->genus->okp(ego->desc.r2hc, p->I, p->O, p->O
+					      + ioffset(ego->kind, ego->sz, p->sz->dims[0].os),
+					      p->sz->dims[0].is,
+					      p->sz->dims[0].os, -p->sz->dims[0].os,
+					      vl, ivs, ovs))
+	       && (!HC2R_KINDP(ego->kind) ||
+		   ego->desc.hc2r->genus->okp(ego->desc.hc2r, p->I, p->I
+					      + ioffset(ego->kind, ego->sz, p->sz->dims[0].is), p->O,
+					      p->sz->dims[0].is, -p->sz->dims[0].is,
+					      p->sz->dims[0].os, 
+					      vl, ivs, ovs))
+	       
+	       && (!R2R_KINDP(ego->kind) ||
+		   ego->desc.r2r->genus->okp(ego->desc.r2r, p->I, p->O,
+					     p->sz->dims[0].is,
+					     p->sz->dims[0].os, 
+					     vl, ivs, ovs))
+	       
+	       && (0
+		   /* can operate out-of-place */
+		   || p->I != p->O
+
+		   /*
+		    * can compute one transform in-place, no matter
+		    * what the strides are.
+		    */
+		   || p->vecsz->rnk == 0
+
+		   /* can operate in-place as long as strides are the same */
+		   || (X(tensor_inplace_strides2)(p->sz, p->vecsz))
+		    )
+	       );
+     }
+
+     return 0;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     P *pln;
+     const problem_rdft *p;
+     iodim *d;
+     int hc2r_kindp, r2r_kindp;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), X(null_awake), print, destroy
+     };
+
+     UNUSED(plnr);
+
+     if (!applicable(ego_, p_))
+          return (plan *)0;
+
+     p = (const problem_rdft *) p_;
+
+     hc2r_kindp = HC2R_KINDP(ego->kind);
+     r2r_kindp = R2R_KINDP(ego->kind);
+
+     pln = MKPLAN_RDFT(P, &padt, 
+		       r2r_kindp ? apply_r2r :
+		       (hc2r_kindp ? apply_hc2r : apply_r2hc));
+
+     d = p->sz->dims;
+
+     pln->k = ego->k;
+     pln->ioffset = ioffset(ego->kind, d[0].n, hc2r_kindp ? d[0].is : d[0].os);
+
+     pln->is = X(mkstride)(ego->sz, hc2r_kindp ? d[0].os : d[0].is);
+     if (r2r_kindp) {
+	  pln->ros = X(mkstride)(ego->sz, d[0].os);
+	  pln->ios = 0;
+     }
+     else {  
+	  int nr = (ego->kind == R2HC || ego->kind == HC2R) 
+	       ?(d[0].n + 2) / 2 : /* R2HCII */ (d[0].n + 1) / 2;
+	  pln->ros = X(mkstride)(nr, hc2r_kindp ? d[0].is : d[0].os);
+	  pln->ios = X(mkstride)(ego->sz - nr + 1, 
+				 hc2r_kindp ? -d[0].is : -d[0].os);
+     }
+
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+
+     pln->slv = ego;
+     X(ops_zero)(&pln->super.super.ops);
+     if (r2r_kindp)
+	  X(ops_madd2)(pln->vl / ego->desc.r2r->genus->vl,
+		       &ego->desc.r2r->ops,
+		       &pln->super.super.ops);
+     else if (hc2r_kindp)
+	  X(ops_madd2)(pln->vl / ego->desc.hc2r->genus->vl,
+		       &ego->desc.hc2r->ops,
+		       &pln->super.super.ops);
+     else
+	  X(ops_madd2)(pln->vl / ego->desc.r2hc->genus->vl,
+		       &ego->desc.r2hc->ops,
+		       &pln->super.super.ops);
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+solver *X(mksolver_rdft_r2hc_direct)(kr2hc k, const kr2hc_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->k.r2hc = k;
+     slv->desc.r2hc = desc;
+     slv->sz = desc->sz;
+     slv->nam = desc->nam;
+     slv->kind = desc->genus->kind;
+     return &(slv->super);
+}
+
+solver *X(mksolver_rdft_hc2r_direct)(khc2r k, const khc2r_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->k.hc2r = k;
+     slv->desc.hc2r = desc;
+     slv->sz = desc->sz;
+     slv->nam = desc->nam;
+     slv->kind = desc->genus->kind;
+     return &(slv->super);
+}
+
+solver *X(mksolver_rdft_r2r_direct)(kr2r k, const kr2r_desc *desc)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->k.r2r = k;
+     slv->desc.r2r = desc;
+     slv->sz = desc->sz;
+     slv->nam = desc->nam;
+     slv->kind = desc->kind;
+     return &(slv->super);
+}
+
diff --git a/src/fftw3/rdft/rgeneric.c b/src/fftw3/rdft/rgeneric.c
new file mode 100644
index 0000000..ebc48e9
--- /dev/null
+++ b/src/fftw3/rdft/rgeneric.c
@@ -0,0 +1,371 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include "rdft.h"
+
+typedef struct {
+     solver super;
+     rdft_kind kind;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cld;
+     twid *td;
+     int os;
+     int r, m;
+     rdft_kind kind;
+} P;
+
+/***************************************************************************/
+
+static void apply_dit(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int n, m, r;
+     int i, j, k;
+     int os, osm;
+     E *buf;
+     const R *W;
+     R *X, *YO, *YI;
+     E rsum, isum;
+     int wp, wincr;
+
+     {
+	  plan_rdft *cld = (plan_rdft *) ego->cld;
+	  cld->apply((plan *) cld, I, O);
+     }
+
+     r = ego->r;
+
+     STACK_MALLOC(E *, buf, r * 2 * sizeof(E));
+     
+     osm = (m = ego->m) * (os = ego->os);
+     n = m * r;
+     W = ego->td->W;
+
+     X = O;
+     YO = O + r * osm;
+     YI = O + osm;
+
+     /* compute the transform of the r 0th elements (which are real) */
+     for (i = 0; i + i < r; ++i) {
+	  rsum = K(0.0);
+	  isum = K(0.0);
+	  wincr = m * i;
+	  for (j = 0, wp = 0; j < r; ++j) {
+	       E tw_r = W[2*wp];
+	       E tw_i = W[2*wp+1] ;
+	       E re = X[j * osm];
+	       rsum += re * tw_r;
+	       isum += re * tw_i;
+	       wp += wincr;
+	       if (wp >= n)
+		    wp -= n;
+	  }
+	  buf[2*i] = rsum;
+	  buf[2*i+1] = isum;
+     }
+
+     /* store the transform back onto the A array */
+     X[0] = buf[0];
+     for (i = 1; i + i < r; ++i) {
+	  X[i * osm] = buf[2*i];
+	  YO[-i * osm] = buf[2*i+1];
+     }
+
+     X += os;
+     YI -= os;
+     YO -= os;
+
+     /* compute the transform of the middle elements (which are complex) */
+     for (k = 1; k + k < m; ++k, X += os, YI -= os, YO -= os) {
+	  for (i = 0; i < r; ++i) {
+	       rsum = K(0.0);
+	       isum = K(0.0);
+	       wincr = k + m * i;
+	       for (j = 0, wp = 0; j < r; ++j) {
+		    E tw_r = W[2*wp];
+		    E tw_i = W[2*wp+1] ;
+		    E re = X[j * osm];
+		    E im = YI[j * osm];
+		    rsum += re * tw_r - im * tw_i;
+		    isum += re * tw_i + im * tw_r;
+		    wp += wincr;
+		    if (wp >= n)
+			 wp -= n;
+	       }
+	       buf[2*i] = rsum;
+	       buf[2*i+1] = isum;
+	  }
+
+	  /* store the transform back onto the A array */
+	  for (i = 0; i + i < r; ++i) {
+	       X[i * osm] = buf[2*i];
+	       YO[-i * osm] = buf[2*i+1];
+	  }
+	  for (; i < r; ++i) {
+	       X[i * osm] = -buf[2*i+1];
+	       YO[-i * osm] = buf[2*i];
+	  }
+     }
+
+     /* no final element, since m is odd */
+
+     STACK_FREE(buf);
+}
+
+static void apply_dif(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int n, m, r;
+     int i, j, k;
+     int is, ism;
+     E *buf;
+     const R *W;
+     R *X, *YO, *YI;
+     E rsum, isum;
+     int wp, wincr;
+
+     r = ego->r;
+
+     STACK_MALLOC(E *, buf, r * 2 * sizeof(E));
+     
+     ism = (m = ego->m) * (is = ego->os);
+     n = m * r;
+     W = ego->td->W;
+
+     X = I;
+     YI = I + r * ism;
+     YO = I + ism;
+
+     /* 
+      * compute the transform of the r 0th elements (which are halfcomplex)
+      * yielding real numbers
+      */
+     /* copy the input into the temporary array */
+     buf[0] = X[0];
+     for (i = 1; i + i < r; ++i) {
+	  buf[2*i] = X[i * ism];
+	  buf[2*i+1] = YI[-i * ism];
+     }
+
+     for (i = 0; i < r; ++i) {
+	  rsum = K(0.0);
+	  wincr = m * i;
+	  for (j = 1, wp = wincr; j + j < r; ++j) {
+	       E tw_r = W[2*wp];
+	       E tw_i = W[2*wp+1];
+	       E re = buf[2*j];
+	       E im = buf[2*j+1];
+	       rsum += re * tw_r + im * tw_i;
+	       wp += wincr;
+	       if (wp >= n)
+		    wp -= n;
+	  }
+	  X[i * ism] = K(2.0) * rsum + buf[0];
+     }
+
+     X += is;
+     YI -= is;
+     YO -= is;
+
+     /* compute the transform of the middle elements (which are complex) */
+     for (k = 1; k + k < m; ++k, X += is, YI -= is, YO -= is) {
+	  /* copy the input into the temporary array */
+	  for (i = 0; i + i < r; ++i) {
+	       buf[2*i] = X[i * ism];
+	       buf[2*i+1] = YI[-i * ism];
+	  }
+	  for (; i < r; ++i) {
+	       buf[2*i+1] = -X[i * ism];
+	       buf[2*i] = YI[-i * ism];
+	  }
+
+	  for (i = 0; i < r; ++i) {
+	       rsum = K(0.0);
+	       isum = K(0.0);
+	       wincr = m * i;
+	       for (j = 0, wp = k * i; j < r; ++j) {
+		    E tw_r = W[2*wp];
+		    E tw_i = W[2*wp+1];
+		    E re = buf[2*j];
+		    E im = buf[2*j+1];
+		    rsum += re * tw_r + im * tw_i;
+		    isum += im * tw_r - re * tw_i;
+		    wp += wincr;
+		    if (wp >= n)
+			 wp -= n;
+	       }
+	       X[i * ism] = rsum;
+	       YO[i * ism] = isum;
+	  }
+     }
+
+     /* no final element, since m is odd */
+
+     STACK_FREE(buf);
+
+     {
+	  plan_rdft *cld = (plan_rdft *) ego->cld;
+	  cld->apply((plan *) cld, I, O);
+     }
+
+}
+
+/***************************************************************************/
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     static const tw_instr generic_tw[] = {
+	  { TW_GENERIC, 0, 0 },
+	  { TW_NEXT, 1, 0 }
+     };
+
+     AWAKE(ego->cld, flg);
+     /* FIXME: can we get away with fewer twiddles? */
+     X(twiddle_awake)(flg, &ego->td, generic_tw,
+		      ego->r * ego->m, ego->r, ego->m);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+
+     p->print(p, "(rdft-generic-%s-%d%(%p%))", 
+	      ego->kind == R2HC ? "r2hc-dit" : "hc2r-dif",
+	      ego->r, ego->cld);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     if (RDFTP(p_)) {
+	  const S *ego = (const S *) ego_;
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk == 0
+		  && p->sz->dims[0].n > 1
+                  && p->sz->dims[0].n % 2 /* ensure r and n/r odd */
+                  && p->kind[0] == ego->kind
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego_, const problem *p_, 
+		      const planner *plnr)
+{
+     if (NO_UGLYP(plnr)) return 0; /* always ugly */
+     if (!applicable0(ego_, p_)) return 0;
+
+     if (NO_LARGE_GENERICP(plnr)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+	  if (X(first_divisor)(p->sz->dims[0].n) >= GENERIC_MIN_BAD) return 0; 
+     }
+     return 1;
+}
+
+static plan *mkplan(const solver *ego, const problem *p_, planner *plnr)
+{
+     const problem_rdft *p = (const problem_rdft *) p_;
+     P *pln = 0;
+     int n, r, m;
+     int is, os;
+     plan *cld = (plan *) 0;
+     problem *cldp;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego, p_, plnr))
+          goto nada;
+
+     n = p->sz->dims[0].n;
+     is = p->sz->dims[0].is;
+     os = p->sz->dims[0].os;
+
+     r = X(first_divisor)(n);
+     m = n / r;
+
+     if (R2HC_KINDP(p->kind[0])) {
+	  cldp = X(mkproblem_rdft_d)(X(mktensor_1d)(m, r * is, os),
+				     X(mktensor_1d)(r, is, m * os),
+				     p->I, p->O, p->kind);
+     }
+     else {
+	  cldp = X(mkproblem_rdft_d)(X(mktensor_1d)(m, is, r * os),
+				     X(mktensor_1d)(r, m * is, os),
+				     p->I, p->O, p->kind);
+     }
+     if (!(cld = X(mkplan_d)(plnr, cldp))) goto nada;
+
+     pln = MKPLAN_RDFT(P, &padt, R2HC_KINDP(p->kind[0]) ? apply_dit:apply_dif);
+
+     pln->os = R2HC_KINDP(p->kind[0]) ? os : is;
+     pln->r = r;
+     pln->m = m;
+     pln->cld = cld;
+     pln->td = 0;
+     pln->kind = p->kind[0];
+
+     X(ops_zero)(&pln->super.super.ops);
+     pln->super.super.ops.add = 4 * r * r;
+     pln->super.super.ops.mul = 4 * r * r;
+     /* loads + stores, minus loads + stores for all DIT codelets */
+     pln->super.super.ops.other = 4 * r + 4 * r * r - (6*r - 2);
+     X(ops_madd)((m - 1)/2, &pln->super.super.ops, &cld->ops,
+		 &pln->super.super.ops);
+     pln->super.super.ops.add += 2 * r * r;
+     pln->super.super.ops.mul += 2 * r * r;
+     pln->super.super.ops.other += 3 * r + 3 * r * r - 2*r;
+
+     return &(pln->super.super);
+
+ nada:
+     X(plan_destroy_internal)(cld);
+     X(ifree0)(pln);
+     return (plan *) 0;
+}
+
+/* constructors */
+
+static solver *mksolver(rdft_kind kind)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->kind = kind;
+     return &(slv->super);
+}
+
+void X(rdft_generic_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver(R2HC));
+     REGISTER_SOLVER(p, mksolver(HC2R));
+}
diff --git a/src/fftw3/rdft/rindirect.c b/src/fftw3/rdft/rindirect.c
new file mode 100644
index 0000000..a53020f
--- /dev/null
+++ b/src/fftw3/rdft/rindirect.c
@@ -0,0 +1,242 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rindirect.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+
+/* solvers/plans for vectors of small RDFT's that cannot be done
+   in-place directly.  Use a rank-0 plan to rearrange the data
+   before or after the transform.  Can also change an out-of-place
+   plan into a copy + in-place (where the in-place transform
+   is e.g. unit stride). */
+
+/* FIXME: merge with rank-geq2.c(?), since this is just a special case
+   of a rank split where the first/second transform has rank 0. */
+
+#include "rdft.h"
+
+typedef problem *(*mkcld_t) (const problem_rdft *p);
+
+typedef struct {
+     rdftapply apply;
+     problem *(*mkcld)(const problem_rdft *p);
+     const char *nam;
+} ndrct_adt;
+
+typedef struct {
+     solver super;
+     const ndrct_adt *adt;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cldcpy, *cld;
+     const S *slv;
+} P;
+
+/*-----------------------------------------------------------------------*/
+/* first rearrange, then transform */
+static void apply_before(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+
+     {
+          plan_rdft *cldcpy = (plan_rdft *) ego->cldcpy;
+          cldcpy->apply(ego->cldcpy, I, O);
+     }
+     {
+          plan_rdft *cld = (plan_rdft *) ego->cld;
+          cld->apply(ego->cld, O, O);
+     }
+}
+
+static problem *mkcld_before(const problem_rdft *p)
+{
+     return X(mkproblem_rdft_d)(X(tensor_copy_inplace)(p->sz, INPLACE_OS),
+				X(tensor_copy_inplace)(p->vecsz, INPLACE_OS),
+				p->O, p->O, p->kind);
+}
+
+static const ndrct_adt adt_before =
+{
+     apply_before, mkcld_before, "rdft-indirect-before"
+};
+
+/*-----------------------------------------------------------------------*/
+/* first transform, then rearrange */
+
+static void apply_after(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+
+     {
+          plan_rdft *cld = (plan_rdft *) ego->cld;
+          cld->apply(ego->cld, I, I);
+     }
+     {
+          plan_rdft *cldcpy = (plan_rdft *) ego->cldcpy;
+          cldcpy->apply(ego->cldcpy, I, O);
+     }
+}
+
+static problem *mkcld_after(const problem_rdft *p)
+{
+     return X(mkproblem_rdft_d)(X(tensor_copy_inplace)(p->sz, INPLACE_IS),
+				X(tensor_copy_inplace)(p->vecsz, INPLACE_IS),
+				p->I, p->I, p->kind);
+}
+
+static const ndrct_adt adt_after =
+{
+     apply_after, mkcld_after, "rdft-indirect-after"
+};
+
+/*-----------------------------------------------------------------------*/
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+     X(plan_destroy_internal)(ego->cldcpy);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cldcpy, flg);
+     AWAKE(ego->cld, flg);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     const S *s = ego->slv;
+     p->print(p, "(%s%(%p%)%(%p%))", s->adt->nam, ego->cld, ego->cldcpy);
+}
+
+static int applicable0(const solver *ego_, const problem *p_,
+		       const planner *plnr)
+{
+     if (RDFTP(p_)) {
+	  const S *ego = (const S *) ego_;
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+                  && FINITE_RNK(p->vecsz->rnk)
+
+                  /* problem must be a nontrivial transform, not just a copy */
+                  && p->sz->rnk > 0
+
+                  && (0
+
+		      /* problem must be in-place & require some
+		         rearrangement of the data */
+		      || (p->I == p->O
+			  && !(X(tensor_inplace_strides2)(p->sz, p->vecsz)))
+
+		      /* or problem must be out of place, transforming
+			 from stride 1/2 to bigger stride, for apply_after */
+		      || (p->I != p->O && ego->adt->apply == apply_after
+			  && DESTROY_INPUTP(plnr)
+			  && X(tensor_min_istride)(p->sz) <= 2
+			  && X(tensor_min_ostride)(p->sz) > 2)
+			  
+		      /* or problem must be out of place, transforming
+			 to stride 1/2 from bigger stride, for apply_before */
+		      || (p->I != p->O && ego->adt->apply == apply_before
+			  && X(tensor_min_ostride)(p->sz) <= 2
+			  && X(tensor_min_istride)(p->sz) > 2)
+			  
+		       )
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego_, const problem *p_,
+		      const planner *plnr)
+{
+     if (!applicable0(ego_, p_, plnr)) return 0;
+	  
+     if (NO_INDIRECT_OP_P(plnr)) {
+	  const problem_rdft *p = (const problem_rdft *)p_;
+	  if (p->I != p->O) return 0;
+     }
+
+     return 1;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const problem_rdft *p = (const problem_rdft *) p_;
+     const S *ego = (const S *) ego_;
+     P *pln;
+     plan *cld = 0, *cldcpy = 0;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+          return (plan *) 0;
+
+     plnr->planner_flags |= NO_BUFFERING;
+
+     cldcpy = X(mkplan_d)(plnr, 
+			  X(mkproblem_rdft_d)(
+			       X(mktensor_0d)(),
+			       X(tensor_append)(p->vecsz, p->sz),
+			       p->I, p->O, (rdft_kind *) 0));
+     if (!cldcpy) goto nada;
+
+     cld = X(mkplan_d)(plnr, ego->adt->mkcld(p));
+     if (!cld) goto nada;
+
+     pln = MKPLAN_RDFT(P, &padt, ego->adt->apply);
+     pln->cld = cld;
+     pln->cldcpy = cldcpy;
+     pln->slv = ego;
+     X(ops_add)(&cld->ops, &cldcpy->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+
+ nada:
+     X(plan_destroy_internal)(cld);
+     X(plan_destroy_internal)(cldcpy);
+     return (plan *)0;
+}
+
+static solver *mksolver(const ndrct_adt *adt)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->adt = adt;
+     return &(slv->super);
+}
+
+void X(rdft_indirect_register)(planner *p)
+{
+     unsigned i;
+     static const ndrct_adt *const adts[] = {
+	  &adt_before, &adt_after
+     };
+
+     for (i = 0; i < sizeof(adts) / sizeof(adts[0]); ++i)
+          REGISTER_SOLVER(p, mksolver(adts[i]));
+}
diff --git a/src/fftw3/rdft/rnop.c b/src/fftw3/rdft/rnop.c
new file mode 100644
index 0000000..e784a5a
--- /dev/null
+++ b/src/fftw3/rdft/rnop.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rnop.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* plans for vrank -infty RDFTs (nothing to do) */
+
+#include "rdft.h"
+
+static void apply(const plan *ego_, R *I, R *O)
+{
+     UNUSED(ego_);
+     UNUSED(I);
+     UNUSED(O);
+}
+
+static int applicable(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return 0
+	       /* case 1 : -infty vector rank */
+	       || (p->vecsz->rnk == RNK_MINFTY)
+
+	       /* case 2 : rank-0 in-place rdft */
+	       || (1
+		   && p->sz->rnk == 0
+		   && FINITE_RNK(p->vecsz->rnk)
+		   && p->O == p->I
+		   && X(tensor_inplace_strides)(p->vecsz)
+                    );
+     }
+     return 0;
+}
+
+static void print(const plan *ego, printer *p)
+{
+     UNUSED(ego);
+     p->print(p, "(rdft-nop)");
+}
+
+static plan *mkplan(const solver *ego, const problem *p, planner *plnr)
+{
+     static const plan_adt padt = {
+	  X(rdft_solve), X(null_awake), print, X(plan_null_destroy)
+     };
+     plan_rdft *pln;
+
+     UNUSED(plnr);
+
+     if (!applicable(ego, p))
+          return (plan *) 0;
+     pln = MKPLAN_RDFT(plan_rdft, &padt, apply);
+     X(ops_zero)(&pln->super.ops);
+
+     return &(pln->super);
+}
+
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     return MKSOLVER(solver, &sadt);
+}
+
+void X(rdft_nop_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/rdft/rplan.c b/src/fftw3/rdft/rplan.c
new file mode 100644
index 0000000..066e925
--- /dev/null
+++ b/src/fftw3/rdft/rplan.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rplan.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+plan *X(mkplan_rdft)(size_t size, const plan_adt *adt, rdftapply apply)
+{
+     plan_rdft *ego;
+
+     ego = (plan_rdft *) X(mkplan)(size, adt);
+     ego->apply = apply;
+
+     return &(ego->super);
+}
diff --git a/src/fftw3/rdft/rproblem.c b/src/fftw3/rdft/rproblem.c
new file mode 100644
index 0000000..0e3441c
--- /dev/null
+++ b/src/fftw3/rdft/rproblem.c
@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rproblem.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+#include <stddef.h>
+
+static void destroy(problem *ego_)
+{
+     problem_rdft *ego = (problem_rdft *) ego_;
+#if !defined(STRUCT_HACK_C99) && !defined(STRUCT_HACK_KR)
+     X(ifree0)(ego->kind);
+#endif
+     X(tensor_destroy2)(ego->vecsz, ego->sz);
+     X(ifree)(ego_);
+}
+
+static void kind_hash(md5 *m, const rdft_kind *kind, int rnk)
+{
+     int i;
+     for (i = 0; i < rnk; ++i)
+	  X(md5int)(m, kind[i]);
+}
+
+static void hash(const problem *p_, md5 *m)
+{
+     const problem_rdft *p = (const problem_rdft *) p_;
+     X(md5puts)(m, "rdft");
+     X(md5int)(m, p->I == p->O);
+     kind_hash(m, p->kind, p->sz->rnk);
+     X(md5int)(m, X(alignment_of)(p->I));
+     X(md5int)(m, X(alignment_of)(p->O));
+     X(tensor_md5)(m, p->sz);
+     X(tensor_md5)(m, p->vecsz);
+}
+
+static void recur(const iodim *dims, int rnk, R *I)
+{
+     if (rnk == RNK_MINFTY)
+          return;
+     else if (rnk == 0)
+          I[0] = K(0.0);
+     else if (rnk > 0) {
+          int i, n = dims[0].n;
+          int is = dims[0].is;
+
+	  if (rnk == 1) {
+	       /* this case is redundant but faster */
+	       for (i = 0; i < n; ++i)
+		    I[i * is] = K(0.0);
+	  } else {
+	       for (i = 0; i < n; ++i)
+		    recur(dims + 1, rnk - 1, I + i * is);
+	  }
+     }
+}
+
+void X(rdft_zerotens)(tensor *sz, R *I)
+{
+     recur(sz->dims, sz->rnk, I);
+}
+
+#define KSTR_LEN 8
+
+const char *X(rdft_kind_str)(rdft_kind kind)
+{
+     static const char kstr[][KSTR_LEN] = {
+	  "r2hc", "r2hc01", "r2hc10", "r2hc11",
+	  "hc2r", "hc2r01", "hc2r10", "hc2r11",
+	  "dht",
+	  "redft00", "redft01", "redft10", "redft11",
+	  "rodft00", "rodft01", "rodft10", "rodft11"
+     };
+     A(kind >= 0 && kind < sizeof(kstr) / KSTR_LEN);
+     return kstr[kind];
+}
+
+static void print(problem *ego_, printer *p)
+{
+     const problem_rdft *ego = (const problem_rdft *) ego_;
+     int i;
+     p->print(p, "(rdft %d %td %T %T", 
+	      X(alignment_of)(ego->I),
+	      ego->O - ego->I, 
+	      ego->sz,
+	      ego->vecsz);
+     for (i = 0; i < ego->sz->rnk; ++i)
+	  p->print(p, " %d", (int)ego->kind[i]);
+     p->print(p, ")");
+}
+
+static void zero(const problem *ego_)
+{
+     const problem_rdft *ego = (const problem_rdft *) ego_;
+     tensor *sz = X(tensor_append)(ego->vecsz, ego->sz);
+     X(rdft_zerotens)(sz, UNTAINT(ego->I));
+     X(tensor_destroy)(sz);
+}
+
+static const problem_adt padt =
+{
+     hash,
+     zero,
+     print,
+     destroy
+};
+
+int X(problem_rdft_p)(const problem *p)
+{
+     return (p->adt == &padt);
+}
+
+/* Dimensions of size 1 that are not REDFT/RODFT are no-ops and can be
+   eliminated.  REDFT/RODFT unit dimensions often have factors of 2.0
+   and suchlike from normalization and phases, although in principle
+   these constant factors from different dimensions could be combined. */
+static int nontrivial(const iodim *d, rdft_kind kind)
+{
+     return (d->n > 1 || kind == R2HC11 || kind == HC2R11
+	     || (REODFT_KINDP(kind) && kind != REDFT01 && kind != RODFT01));
+}
+
+problem *X(mkproblem_rdft)(const tensor *sz, const tensor *vecsz,
+			   R *I, R *O, const rdft_kind *kind)
+{
+     problem_rdft *ego;
+     int rnk = sz->rnk;
+     int i;
+
+     A(X(tensor_kosherp)(sz));
+     A(X(tensor_kosherp)(vecsz));
+     A(FINITE_RNK(sz->rnk));
+
+     if (UNTAINT(I) == UNTAINT(O))
+	  I = O = JOIN_TAINT(I, O);
+
+     for (i = rnk = 0; i < sz->rnk; ++i) {
+          A(sz->dims[i].n > 0);
+          if (nontrivial(sz->dims + i, kind[i]))
+               ++rnk;
+     }
+
+#if defined(STRUCT_HACK_KR)
+     ego = (problem_rdft *) X(mkproblem)(sizeof(problem_rdft)
+					 + sizeof(rdft_kind)
+					 * (rnk > 0 ? rnk - 1 : 0), &padt);
+#elif defined(STRUCT_HACK_C99)
+     ego = (problem_rdft *) X(mkproblem)(sizeof(problem_rdft)
+					 + sizeof(rdft_kind) * rnk, &padt);
+#else
+     ego = (problem_rdft *) X(mkproblem)(sizeof(problem_rdft), &padt);
+     ego->kind = (rdft_kind *) MALLOC(sizeof(rdft_kind) * rnk, PROBLEMS);
+#endif
+
+     /* do compression and sorting as in X(tensor_compress), but take
+	transform kind into account (sigh) */
+     ego->sz = X(mktensor)(rnk);
+     for (i = rnk = 0; i < sz->rnk; ++i) {
+          if (nontrivial(sz->dims + i, kind[i])) {
+	       ego->kind[rnk] = kind[i];
+               ego->sz->dims[rnk++] = sz->dims[i];
+	  }
+     }
+     for (i = 0; i + 1 < rnk; ++i) {
+	  int j;
+	  for (j = i + 1; j < rnk; ++j)
+	       if (X(dimcmp)(ego->sz->dims + i, ego->sz->dims + j) > 0) {
+		    iodim dswap;
+		    rdft_kind kswap;
+		    dswap = ego->sz->dims[i];
+		    ego->sz->dims[i] = ego->sz->dims[j];
+		    ego->sz->dims[j] = dswap;
+		    kswap = ego->kind[i];
+		    ego->kind[i] = ego->kind[j];
+		    ego->kind[j] = kswap;
+	       }
+     }
+
+     for (i = 0; i < rnk; ++i)
+	  if (ego->sz->dims[i].n == 2 && (ego->kind[i] == REDFT00
+					  || ego->kind[i] == DHT
+					  || ego->kind[i] == HC2R))
+	       ego->kind[i] = R2HC; /* size-2 transforms are equivalent */
+
+     ego->vecsz = X(tensor_compress_contiguous)(vecsz);
+     ego->I = I;
+     ego->O = O;
+
+     A(FINITE_RNK(ego->sz->rnk));
+
+     return &(ego->super);
+}
+
+/* Same as X(mkproblem_rdft), but also destroy input tensors. */
+problem *X(mkproblem_rdft_d)(tensor *sz, tensor *vecsz,
+			     R *I, R *O, const rdft_kind *kind)
+{
+     problem *p;
+     p = X(mkproblem_rdft)(sz, vecsz, I, O, kind);
+     X(tensor_destroy2)(vecsz, sz);
+     return p;
+}
+
+/* As above, but for rnk <= 1 only and takes a scalar kind parameter */
+problem *X(mkproblem_rdft_1)(const tensor *sz, const tensor *vecsz,
+			     R *I, R *O, rdft_kind kind)
+{
+     A(sz->rnk <= 1);
+     return X(mkproblem_rdft)(sz, vecsz, I, O, &kind);
+}
+
+problem *X(mkproblem_rdft_1_d)(tensor *sz, tensor *vecsz,
+			       R *I, R *O, rdft_kind kind)
+{
+     A(sz->rnk <= 1);
+     return X(mkproblem_rdft_d)(sz, vecsz, I, O, &kind);
+}
diff --git a/src/fftw3/rdft/rrank-geq2.c b/src/fftw3/rdft/rrank-geq2.c
new file mode 100644
index 0000000..78f359f
--- /dev/null
+++ b/src/fftw3/rdft/rrank-geq2.c
@@ -0,0 +1,216 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rrank-geq2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* plans for RDFT of rank >= 2 (multidimensional) */
+
+/* FIXME: this solver cannot strictly be applied to multidimensional
+   DHTs, since the latter are not separable...up to rnk-1 additional
+   post-processing passes may be required.  See also:
+
+   R. N. Bracewell, O. Buneman, H. Hao, and J. Villasenor, "Fast
+   two-dimensional Hartley transform," Proc. IEEE 74, 1282-1283 (1986).
+
+   H. Hao and R. N. Bracewell, "A three-dimensional DFT algorithm
+   using the fast Hartley transform," Proc. IEEE 75(2), 264-266 (1987).
+*/
+
+#include "rdft.h"
+
+typedef struct {
+     solver super;
+     int spltrnk;
+     const int *buddies;
+     int nbuddies;
+} S;
+
+typedef struct {
+     plan_rdft super;
+
+     plan *cld1, *cld2;
+     const S *solver;
+} P;
+
+/* Compute multi-dimensional RDFT by applying the two cld plans
+   (lower-rnk RDFTs). */
+static void apply(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     plan_rdft *cld1, *cld2;
+
+     cld1 = (plan_rdft *) ego->cld1;
+     cld1->apply(ego->cld1, I, O);
+
+     cld2 = (plan_rdft *) ego->cld2;
+     cld2->apply(ego->cld2, O, O);
+}
+
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cld1, flg);
+     AWAKE(ego->cld2, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld2);
+     X(plan_destroy_internal)(ego->cld1);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     const S *s = ego->solver;
+     p->print(p, "(rdft-rank>=2/%d%(%p%)%(%p%))",
+	      s->spltrnk, ego->cld1, ego->cld2);
+}
+
+static int picksplit(const S *ego, const tensor *sz, int *rp)
+{
+     A(sz->rnk > 1); /* cannot split rnk <= 1 */
+     if (!X(pickdim)(ego->spltrnk, ego->buddies, ego->nbuddies, sz, 1, rp))
+	  return 0;
+     *rp += 1; /* convert from dim. index to rank */
+     if (*rp >= sz->rnk) /* split must reduce rank */
+	  return 0;
+     return 1;
+}
+
+static int applicable0(const solver *ego_, const problem *p_, int *rp)
+{
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          const S *ego = (const S *)ego_;
+          return (1
+                  && p->sz->rnk >= 2
+                  && picksplit(ego, p->sz, rp)
+	       );
+     }
+
+     return 0;
+}
+
+/* TODO: revise this. */
+static int applicable(const solver *ego_, const problem *p_, 
+		      const planner *plnr, int *rp)
+{
+     const S *ego = (const S *)ego_;
+
+     if (!applicable0(ego_, p_, rp)) return 0;
+
+     /* fixed spltrnk (unlike fftw2's spltrnk=1, default buddies[0] is
+        spltrnk=0, which is an asymptotic "theoretical optimum" for
+        an ideal cache; it's equivalent to spltrnk=1 for rnk < 4). */
+     if (NO_RANK_SPLITSP(plnr) && (ego->spltrnk != ego->buddies[0]))
+	  return 0;
+
+     if (NO_UGLYP(plnr)) {
+	  /* Heuristic: if the vector stride is greater than the transform
+	     sz, don't use (prefer to do the vector loop first with a
+	     vrank-geq1 plan). */
+	  const problem_rdft *p = (const problem_rdft *) p_;
+
+	  if (p->vecsz->rnk > 0 &&
+	      X(tensor_min_stride)(p->vecsz) > X(tensor_max_index)(p->sz))
+	       return 0;
+     }
+
+     return 1;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     const problem_rdft *p;
+     P *pln;
+     plan *cld1 = 0, *cld2 = 0;
+     tensor *sz1, *sz2, *vecszi, *sz2i;
+     int spltrnk;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr, &spltrnk))
+          return (plan *) 0;
+
+     p = (const problem_rdft *) p_;
+     X(tensor_split)(p->sz, &sz1, spltrnk, &sz2);
+     vecszi = X(tensor_copy_inplace)(p->vecsz, INPLACE_OS);
+     sz2i = X(tensor_copy_inplace)(sz2, INPLACE_OS);
+
+     cld1 = X(mkplan_d)(plnr, 
+			X(mkproblem_rdft_d)(X(tensor_copy)(sz2),
+					    X(tensor_append)(p->vecsz, sz1),
+					    p->I, p->O, p->kind + spltrnk));
+     if (!cld1) goto nada;
+
+     cld2 = X(mkplan_d)(plnr, 
+			X(mkproblem_rdft_d)(
+			     X(tensor_copy_inplace)(sz1, INPLACE_OS),
+			     X(tensor_append)(vecszi, sz2i),
+			     p->O, p->O, p->kind));
+     if (!cld2) goto nada;
+
+     pln = MKPLAN_RDFT(P, &padt, apply);
+
+     pln->cld1 = cld1;
+     pln->cld2 = cld2;
+
+     pln->solver = ego;
+     X(ops_add)(&cld1->ops, &cld2->ops, &pln->super.super.ops);
+
+     X(tensor_destroy4)(sz2, sz1, vecszi, sz2i);
+
+     return &(pln->super.super);
+
+ nada:
+     X(plan_destroy_internal)(cld2);
+     X(plan_destroy_internal)(cld1);
+     X(tensor_destroy4)(sz2, sz1, vecszi, sz2i);
+     return (plan *) 0;
+}
+
+static solver *mksolver(int spltrnk, const int *buddies, int nbuddies)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->spltrnk = spltrnk;
+     slv->buddies = buddies;
+     slv->nbuddies = nbuddies;
+     return &(slv->super);
+}
+
+void X(rdft_rank_geq2_register)(planner *p)
+{
+     int i;
+     static const int buddies[] = { 0, 1, -2 };
+
+     const int nbuddies = sizeof(buddies) / sizeof(buddies[0]);
+
+     for (i = 0; i < nbuddies; ++i)
+          REGISTER_SOLVER(p, mksolver(buddies[i], buddies, nbuddies));
+
+     /* FIXME: Should we try more buddies?  See also dft/rank-geq2. */
+}
diff --git a/src/fftw3/rdft/rrank0.c b/src/fftw3/rdft/rrank0.c
new file mode 100644
index 0000000..4e212ab
--- /dev/null
+++ b/src/fftw3/rdft/rrank0.c
@@ -0,0 +1,194 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rrank0.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+/* plans for rank-0 RDFTs (copy operations) */
+
+#include "rdft.h"
+
+#ifdef HAVE_STRING_H
+#include <string.h>		/* for memcpy() */
+#endif
+
+
+typedef struct {
+     rdftapply apply;
+     int (*applicable)(const problem_rdft *p);
+     const char *nam;
+} rnk0adt;
+
+typedef struct {
+     solver super;
+     const rnk0adt *adt;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     int vl;
+     int ivs, ovs;
+     const S *slv;
+} P;
+
+/* generic applicability function */
+static int applicable(const solver *ego_, const problem *p_)
+{
+     if (RDFTP(p_)) {
+          const S *ego = (const S *) ego_;
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->I != p->O
+                  && p->sz->rnk == 0
+                  && ego->adt->applicable(p)
+	       );
+     }
+     return 0;
+}
+
+/*-----------------------------------------------------------------------*/
+/* rank-0 rdft, vl == 1: just a copy */
+static void apply_1(const plan *ego_, R *I, R *O)
+{
+     UNUSED(ego_);
+     *O = *I;
+}
+
+static int applicable_1(const problem_rdft *p)
+{
+     return (p->vecsz->rnk == 0);
+}
+
+static const rnk0adt adt_cpy1 =
+{
+     apply_1, applicable_1, "rdft-rank0-cpy1"
+};
+
+/*-----------------------------------------------------------------------*/
+/* rank-0 rdft, vl > 1: just a copy loop (unroll 4) */
+static void apply_vec(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int i, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+
+     for (i = 4; i <= vl; i += 4) {
+          R r0, r1, r2, r3;
+          r0 = *I; I += ivs;
+          r1 = *I; I += ivs;
+          r2 = *I; I += ivs;
+          r3 = *I; I += ivs;
+          *O = r0; O += ovs;
+          *O = r1; O += ovs;
+          *O = r2; O += ovs;
+	  *O = r3; O += ovs;
+     }
+     for (; i < vl + 4; ++i) {
+          R r0;
+          r0 = *I; I += ivs;
+          *O = r0; O += ovs;
+     }
+}
+
+static int applicable_vec(const problem_rdft *p)
+{
+     return (p->vecsz->rnk == 1 && p->O != p->I);
+}
+
+static const rnk0adt adt_vec =
+{
+     apply_vec, applicable_vec, "rdft-rank0-vec"
+};
+
+/*-----------------------------------------------------------------------*/
+/* rank-0 rdft, vl > 1, [io]vs == 1, using memcpy */
+static void apply_io1(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int vl = ego->vl;
+     memcpy(O, I, vl * sizeof(R));
+}
+
+static int applicable_io1(const problem_rdft *p)
+{
+     return (1
+             && applicable_vec(p)
+             && p->vecsz->dims[0].is == 1
+             && p->vecsz->dims[0].os == 1
+	  );
+}
+
+static const rnk0adt adt_io1 =
+{
+     apply_io1, applicable_io1, "rdft-rank0-io1-memcpy"
+};
+
+/*-----------------------------------------------------------------------*/
+/* generic stuff: */
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(%s%v)", ego->slv->adt->nam, ego->vl);
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     const problem_rdft *p;
+     P *pln;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), X(null_awake), print, X(plan_null_destroy)
+     };
+
+     UNUSED(plnr);
+
+     if (!applicable(ego_, p_))
+          return (plan *) 0;
+
+     p = (const problem_rdft *) p_;
+     pln = MKPLAN_RDFT(P, &padt, ego->adt->apply);
+
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     pln->slv = ego;
+
+     /* vl loads, vl stores */
+     X(ops_other)(2 * pln->vl, &pln->super.super.ops);
+     return &(pln->super.super);
+}
+
+static solver *mksolver(const rnk0adt *adt)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->adt = adt;
+     return &(slv->super);
+}
+
+void X(rdft_rank0_register)(planner *p)
+{
+     unsigned i;
+     static const rnk0adt *const adts[] = {
+	  &adt_cpy1, &adt_vec, &adt_io1
+     };
+
+     for (i = 0; i < sizeof(adts) / sizeof(adts[0]); ++i)
+          REGISTER_SOLVER(p, mksolver(adts[i]));
+}
diff --git a/src/fftw3/rdft/rsolve.c b/src/fftw3/rdft/rsolve.c
new file mode 100644
index 0000000..a000a56
--- /dev/null
+++ b/src/fftw3/rdft/rsolve.c
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rsolve.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+/* use the apply() operation for RDFT problems */
+void X(rdft_solve)(const plan *ego_, const problem *p_)
+{
+     const plan_rdft *ego = (const plan_rdft *) ego_;
+     const problem_rdft *p = (const problem_rdft *) p_;
+     ego->apply(ego_, UNTAINT(p->I), UNTAINT(p->O));
+}
diff --git a/src/fftw3/rdft/rvrank-geq1.c b/src/fftw3/rdft/rvrank-geq1.c
new file mode 100644
index 0000000..2bac2d5
--- /dev/null
+++ b/src/fftw3/rdft/rvrank-geq1.c
@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rvrank-geq1.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+
+/* Plans for handling vector transform loops.  These are *just* the
+   loops, and rely on child plans for the actual RDFTs.
+ 
+   They form a wrapper around solvers that don't have apply functions
+   for non-null vectors.
+ 
+   vrank-geq1 plans also recursively handle the case of multi-dimensional
+   vectors, obviating the need for most solvers to deal with this.  We
+   can also play games here, such as reordering the vector loops.
+ 
+   Each vrank-geq1 plan reduces the vector rank by 1, picking out a
+   dimension determined by the vecloop_dim field of the solver. */
+
+#include "rdft.h"
+
+typedef struct {
+     solver super;
+     int vecloop_dim;
+     const int *buddies;
+     int nbuddies;
+} S;
+
+typedef struct {
+     plan_rdft super;
+
+     plan *cld;
+     int vl;
+     int ivs, ovs;
+     const S *solver;
+} P;
+
+static void apply(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int i, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     rdftapply cldapply = ((plan_rdft *) ego->cld)->apply;
+
+     for (i = 0; i < vl; ++i) {
+          cldapply(ego->cld, I + i * ivs, O + i * ovs);
+     }
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cld, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     const S *s = ego->solver;
+     p->print(p, "(rdft-vrank>=1-x%d/%d%(%p%))",
+	      ego->vl, s->vecloop_dim, ego->cld);
+}
+
+static int pickdim(const S *ego, const tensor *vecsz, int oop, int *dp)
+{
+     return X(pickdim)(ego->vecloop_dim, ego->buddies, ego->nbuddies,
+		       vecsz, oop, dp);
+}
+
+static int applicable0(const solver *ego_, const problem *p_, int *dp)
+{
+     if (RDFTP(p_)) {
+          const S *ego = (const S *) ego_;
+          const problem_rdft *p = (const problem_rdft *) p_;
+
+          return (1
+                  && FINITE_RNK(p->vecsz->rnk)
+                  && p->vecsz->rnk > 0
+                  && pickdim(ego, p->vecsz, p->I != p->O, dp)
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego_, const problem *p_, 
+		      const planner *plnr, int *dp)
+{
+     const S *ego = (const S *)ego_;
+     const problem_rdft *p;
+
+     if (!applicable0(ego_, p_, dp)) return 0;
+
+     /* fftw2 behavior */
+     if (NO_VRANK_SPLITSP(plnr) && (ego->vecloop_dim != ego->buddies[0]))
+	  return 0;
+
+     if (NO_UGLYP(plnr)) {
+	  p = (const problem_rdft *) p_;
+
+	  /* Heuristic: if the transform is multi-dimensional, and the
+	     vector stride is less than the transform size, then we
+	     probably want to use a rank>=2 plan first in order to combine
+	     this vector with the transform-dimension vectors. */
+	  {
+	       iodim *d = p->vecsz->dims + *dp;
+	       if (1
+		   && p->sz->rnk > 1 
+		   && X(imin)(X(iabs)(d->is), X(iabs)(d->os))
+		   < X(tensor_max_index)(p->sz)
+		    )
+		    return 0;
+	  }
+
+	  /* Heuristic: don't use a vrank-geq1 for rank-0 vrank-1
+	     transforms, since this case is better handled by rank-0
+	     solvers. */
+	  if (p->sz->rnk == 0 && p->vecsz->rnk == 1) return 0;
+
+	  /* prefer threaded version */
+	  if (NONTHREADED_ICKYP(plnr)) return 0;
+
+	  /* exploit built-in vecloops of (ugly) r{e,o}dft solvers */
+	  if (p->vecsz->rnk == 1 && p->sz->rnk == 1 
+	      && REODFT_KINDP(p->kind[0]))
+	       return 0;
+     }
+
+     return 1;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     const problem_rdft *p;
+     P *pln;
+     plan *cld;
+     int vdim;
+     iodim *d;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr, &vdim))
+          return (plan *) 0;
+     p = (const problem_rdft *) p_;
+
+     d = p->vecsz->dims + vdim;
+
+     A(d->n > 1); 
+
+     cld = X(mkplan_d)(plnr, 
+		       X(mkproblem_rdft_d)(
+			    X(tensor_copy)(p->sz),
+			    X(tensor_copy_except)(p->vecsz, vdim),
+			    TAINT(p->I, d->is), TAINT(p->O, d->os),
+			    p->kind));
+     if (!cld) return (plan *) 0;
+
+     pln = MKPLAN_RDFT(P, &padt, apply);
+
+     pln->cld = cld;
+     pln->vl = d->n;
+     pln->ivs = d->is;
+     pln->ovs = d->os;
+
+     pln->solver = ego;
+     X(ops_zero)(&pln->super.super.ops);
+     pln->super.super.ops.other = 3.14159; /* magic to prefer codelet loops */
+     X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops);
+     pln->super.super.pcost = pln->vl * cld->pcost;
+
+     return &(pln->super.super);
+}
+
+static solver *mksolver(int vecloop_dim, const int *buddies, int nbuddies)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->vecloop_dim = vecloop_dim;
+     slv->buddies = buddies;
+     slv->nbuddies = nbuddies;
+     return &(slv->super);
+}
+
+void X(rdft_vrank_geq1_register)(planner *p)
+{
+     int i;
+
+     /* FIXME: Should we try other vecloop_dim values? */
+     static const int buddies[] = { 1, -1 };
+
+     const int nbuddies = sizeof(buddies) / sizeof(buddies[0]);
+
+     for (i = 0; i < nbuddies; ++i)
+          REGISTER_SOLVER(p, mksolver(buddies[i], buddies, nbuddies));
+}
diff --git a/src/fftw3/rdft/solve2.c b/src/fftw3/rdft/solve2.c
new file mode 100644
index 0000000..adaee27
--- /dev/null
+++ b/src/fftw3/rdft/solve2.c
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: solve2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+#include "rdft.h"
+
+/* use the apply() operation for RDFT2 problems */
+void X(rdft2_solve)(const plan *ego_, const problem *p_)
+{
+     const plan_rdft2 *ego = (const plan_rdft2 *) ego_;
+     const problem_rdft2 *p = (const problem_rdft2 *) p_;
+     ego->apply(ego_, UNTAINT(p->r), UNTAINT(p->rio), UNTAINT(p->iio));
+}
diff --git a/src/fftw3/rdft/vrank-geq1-rdft2.c b/src/fftw3/rdft/vrank-geq1-rdft2.c
new file mode 100644
index 0000000..0557c13
--- /dev/null
+++ b/src/fftw3/rdft/vrank-geq1-rdft2.c
@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: vrank-geq1-rdft2.c,v 1.1 2008/10/17 06:11:29 scuri Exp $ */
+
+
+/* Plans for handling vector transform loops.  These are *just* the
+   loops, and rely on child plans for the actual RDFT2s.
+ 
+   They form a wrapper around solvers that don't have apply functions
+   for non-null vectors.
+ 
+   vrank-geq1-rdft2 plans also recursively handle the case of
+   multi-dimensional vectors, obviating the need for most solvers to
+   deal with this.  We can also play games here, such as reordering
+   the vector loops.
+ 
+   Each vrank-geq1-rdft2 plan reduces the vector rank by 1, picking out a
+   dimension determined by the vecloop_dim field of the solver. */
+
+#include "rdft.h"
+
+typedef struct {
+     solver super;
+     int vecloop_dim;
+     const int *buddies;
+     int nbuddies;
+} S;
+
+typedef struct {
+     plan_rdft2 super;
+
+     plan *cld;
+     int vl;
+     int ivs, ovs;
+     const S *solver;
+} P;
+
+static void apply(const plan *ego_, R *r, R *rio, R *iio)
+{
+     const P *ego = (const P *) ego_;
+     int i, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     rdft2apply cldapply = ((plan_rdft2 *) ego->cld)->apply;
+
+     for (i = 0; i < vl; ++i) {
+          cldapply(ego->cld, r + i * ivs, rio + i * ovs, iio + i * ovs);
+     }
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cld, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     const S *s = ego->solver;
+     p->print(p, "(rdft2-vrank>=1-x%d/%d%(%p%))",
+	      ego->vl, s->vecloop_dim, ego->cld);
+}
+
+static int pickdim(const S *ego, const tensor *vecsz, int oop, int *dp)
+{
+     return X(pickdim)(ego->vecloop_dim, ego->buddies, ego->nbuddies,
+		       vecsz, oop, dp);
+}
+
+static int applicable0(const solver *ego_, const problem *p_, int *dp)
+{
+     if (RDFT2P(p_)) {
+          const S *ego = (const S *) ego_;
+          const problem_rdft2 *p = (const problem_rdft2 *) p_;
+	  if (FINITE_RNK(p->vecsz->rnk)
+	      && p->vecsz->rnk > 0
+	      && pickdim(ego, p->vecsz, 
+			 p->r != p->rio && p->r != p->iio, dp)) {
+	       if (p->r != p->rio && p->r != p->iio)
+		    return 1;  /* can always operate out-of-place */
+
+	       return(X(rdft2_inplace_strides)(p, *dp));
+	  }
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego_, const problem *p_,
+		      const planner *plnr, int *dp)
+{
+     const S *ego = (const S *)ego_;
+
+     if (!applicable0(ego_, p_, dp)) return 0;
+
+     /* fftw2 behavior */
+     if (NO_VRANK_SPLITSP(plnr) && (ego->vecloop_dim != ego->buddies[0]))
+	  return 0;
+
+     if (NO_UGLYP(plnr)) {
+	  const problem_rdft2 *p = (const problem_rdft2 *) p_;
+	  iodim *d = p->vecsz->dims + *dp;
+	       
+	  /* Heuristic: if the transform is multi-dimensional, and the
+	     vector stride is less than the transform size, then we
+	     probably want to use a rank>=2 plan first in order to combine
+	     this vector with the transform-dimension vectors. */
+	  if (p->sz->rnk > 1
+	      && X(imin)(X(iabs)(d->is), X(iabs)(d->os))
+	      < X(rdft2_tensor_max_index)(p->sz, p->kind)
+	       )
+	       return 0;
+
+	  /* Heuristic: don't use a vrank-geq1 for rank-0 vrank-1
+	     transforms, since this case is better handled by rank-0
+	     solvers. */
+	  if (p->sz->rnk == 0 && p->vecsz->rnk == 1) return 0;
+
+	  if (NONTHREADED_ICKYP(plnr)) 
+	       return 0; /* prefer threaded version */
+     }
+
+     return 1;
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     const S *ego = (const S *) ego_;
+     const problem_rdft2 *p;
+     P *pln;
+     plan *cld;
+     int vdim;
+     iodim *d;
+     int ivs, ovs;
+
+     static const plan_adt padt = {
+	  X(rdft2_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr, &vdim))
+          return (plan *) 0;
+     p = (const problem_rdft2 *) p_;
+
+     d = p->vecsz->dims + vdim;
+
+     A(d->n > 1);  /* or else, p->ri + d->is etc. are invalid */
+
+     X(rdft2_strides)(p->kind, d, &ivs, &ovs);
+
+     cld = X(mkplan_d)(plnr, 
+		       X(mkproblem_rdft2_d)(
+			    X(tensor_copy)(p->sz),
+			    X(tensor_copy_except)(p->vecsz, vdim),
+			    TAINT(p->r, ivs), 
+			    TAINT(p->rio, ovs), TAINT(p->iio, ovs),
+			    p->kind));
+     if (!cld) return (plan *) 0;
+
+     pln = MKPLAN_RDFT2(P, &padt, apply);
+
+     pln->cld = cld;
+     pln->vl = d->n;
+     pln->ivs = ivs;
+     pln->ovs = ovs;
+
+     pln->solver = ego;
+     X(ops_zero)(&pln->super.super.ops);
+     pln->super.super.ops.other = 3.14159; /* magic to prefer codelet loops */
+     X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops);
+     pln->super.super.pcost = pln->vl * cld->pcost;
+
+     return &(pln->super.super);
+}
+
+static solver *mksolver(int vecloop_dim, const int *buddies, int nbuddies)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     slv->vecloop_dim = vecloop_dim;
+     slv->buddies = buddies;
+     slv->nbuddies = nbuddies;
+     return &(slv->super);
+}
+
+void X(rdft2_vrank_geq1_register)(planner *p)
+{
+     int i;
+
+     /* FIXME: Should we try other vecloop_dim values? */
+     static const int buddies[] = { 1, -1 };
+
+     const int nbuddies = sizeof(buddies) / sizeof(buddies[0]);
+
+     for (i = 0; i < nbuddies; ++i)
+          REGISTER_SOLVER(p, mksolver(buddies[i], buddies, nbuddies));
+}
diff --git a/src/fftw3/reodft/redft00e-r2hc-pad.c b/src/fftw3/reodft/redft00e-r2hc-pad.c
new file mode 100644
index 0000000..ec3fa35
--- /dev/null
+++ b/src/fftw3/reodft/redft00e-r2hc-pad.c
@@ -0,0 +1,201 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: redft00e-r2hc-pad.c,v 1.1 2008/10/17 06:13:18 scuri Exp $ */
+
+/* Do a REDFT00 problem via an R2HC problem, padded symmetrically to
+   twice the size.  This is asymptotically a factor of ~2 worse than
+   redft00e-r2hc.c (the algorithm used in e.g. FFTPACK and Numerical
+   Recipes), but we abandoned the latter after we discovered that it
+   has intrinsic accuracy problems. */
+
+#include "reodft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cld, *cldcpy;
+     int is;
+     int n;
+     int vl;
+     int ivs, ovs;
+} P;
+
+static void apply(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is;
+     int i, n = ego->n;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * (2*n), BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  buf[0] = I[0];
+	  for (i = 1; i < n; ++i) {
+	       R a = I[i * is];
+	       buf[i] = a;
+	       buf[2*n - i] = a;
+	  }
+	  buf[i] = I[i * is]; /* i == n, Nyquist */
+	  
+	  /* r2hc transform of size 2*n */
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  /* copy n+1 real numbers (real parts of hc array) from buf to O */
+	  {
+	       plan_rdft *cldcpy = (plan_rdft *) ego->cldcpy;
+	       cldcpy->apply((plan *) cldcpy, buf, O);
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cld, flg);
+     AWAKE(ego->cldcpy, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cldcpy);
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(redft00e-r2hc-pad-%d%v%(%p%)%(%p%))", 
+	      ego->n + 1, ego->vl, ego->cld, ego->cldcpy);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk <= 1
+		  && p->kind[0] == REDFT00
+		  && p->sz->dims[0].n > 1  /* n == 1 is not well-defined */
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p, const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0(ego, p));
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     P *pln;
+     const problem_rdft *p;
+     plan *cld = (plan *) 0, *cldcpy;
+     R *buf = (R *) 0;
+     int n;
+     int vl, ivs, ovs;
+     opcnt ops;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+	  goto nada;
+
+     p = (const problem_rdft *) p_;
+
+     n = p->sz->dims[0].n - 1;
+     A(n > 0);
+     buf = (R *) MALLOC(sizeof(R) * (2*n), BUFFERS);
+
+     cld = X(mkplan_d)(plnr,X(mkproblem_rdft_1_d)(X(mktensor_1d)(2*n,1,1), 
+						  X(mktensor_0d)(), 
+						  buf, buf, R2HC));
+     if (!cld)
+	  goto nada;
+
+     X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs);
+     cldcpy =
+	  X(mkplan_d)(plnr,
+		      X(mkproblem_rdft_1_d)(X(mktensor_0d)(),
+					    X(mktensor_1d)(n+1,1,
+							   p->sz->dims[0].os), 
+					    buf, TAINT(p->O, ovs), R2HC));
+     if (!cldcpy)
+	  goto nada;
+
+     X(ifree)(buf);
+
+     pln = MKPLAN_RDFT(P, &padt, apply);
+
+     pln->n = n;
+     pln->is = p->sz->dims[0].is;
+     pln->cld = cld;
+     pln->cldcpy = cldcpy;
+     pln->vl = vl;
+     pln->ivs = ivs;
+     pln->ovs = ovs;
+     
+     X(ops_zero)(&ops);
+     ops.other = n + 2*n; /* loads + stores (input -> buf) */
+
+     X(ops_zero)(&pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &cldcpy->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+
+ nada:
+     X(ifree0)(buf);
+     if (cld)
+	  X(plan_destroy_internal)(cld);  
+     return (plan *)0;
+}
+
+/* constructor */
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(redft00e_r2hc_pad_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/reodft/redft00e-r2hc.c b/src/fftw3/reodft/redft00e-r2hc.c
new file mode 100644
index 0000000..0cd742f
--- /dev/null
+++ b/src/fftw3/reodft/redft00e-r2hc.c
@@ -0,0 +1,216 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: redft00e-r2hc.c,v 1.1 2008/10/17 06:13:18 scuri Exp $ */
+
+/* Do a REDFT00 problem via an R2HC problem, with some pre/post-processing.
+
+   This code uses the trick from FFTPACK, also documented in a similar
+   form by Numerical Recipes.  Unfortunately, this algorithm seems to
+   have intrinsic numerical problems (similar to those in
+   reodft11e-r2hc.c), possibly due to the fact that it multiplies its
+   input by a cosine, causing a loss of precision near the zero.  For
+   transforms of 16k points, it has already lost three or four decimal
+   places of accuracy, which we deem unacceptable.
+
+   So, we have abandoned this algorithm in favor of the one in
+   redft00-r2hc-pad.c, which unfortunately sacrifices 30-50% in speed.
+   The only other alternative in the literature that does not have
+   similar numerical difficulties seems to be the direct adaptation of
+   the Cooley-Tukey decomposition for symmetric data, but this would
+   require a whole new set of codelets and it's not clear that it's
+   worth it at this point. */
+
+#include "reodft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cld;
+     twid *td;
+     int is, os;
+     int n;
+     int vl;
+     int ivs, ovs;
+} P;
+
+static void apply(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *W = ego->td->W;
+     R *buf;
+     E csum;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  buf[0] = I[0] + I[is * n];
+	  csum = I[0] - I[is * n];
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b, apb, amb;
+	       a = I[is * i];
+	       b = I[is * (n - i)];
+	       csum += W[2*i] * (amb = K(2.0)*(a - b));
+	       amb = W[2*i+1] * amb;
+	       apb = (a + b);
+	       buf[i] = apb - amb;
+	       buf[n - i] = apb + amb;
+	  }
+	  if (i == n - i) {
+	       buf[i] = K(2.0) * I[is * i];
+	  }
+	  
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  /* FIXME: use recursive/cascade summation for better stability? */
+	  O[0] = buf[0];
+	  O[os] = csum;
+	  for (i = 1; i + i < n; ++i) {
+	       int k = i + i;
+	       O[os * k] = buf[i];
+	       O[os * (k + 1)] = O[os * (k - 1)] - buf[n - i];
+	  }
+	  if (i + i == n) {
+	       O[os * n] = buf[i];
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     static const tw_instr redft00e_tw[] = {
+          { TW_COS, 0, 1 },
+          { TW_SIN, 0, 1 },
+          { TW_NEXT, 1, 0 }
+     };
+
+     AWAKE(ego->cld, flg);
+     X(twiddle_awake)(flg, &ego->td, redft00e_tw, 2*ego->n, 1, (ego->n+1)/2);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(redft00e-r2hc-%d%v%(%p%))", ego->n + 1, ego->vl, ego->cld);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk <= 1
+		  && p->kind[0] == REDFT00
+		  && p->sz->dims[0].n > 1  /* n == 1 is not well-defined */
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p, const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0(ego, p));
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     P *pln;
+     const problem_rdft *p;
+     plan *cld;
+     R *buf;
+     int n;
+     opcnt ops;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+          return (plan *)0;
+
+     p = (const problem_rdft *) p_;
+
+     n = p->sz->dims[0].n - 1;
+     A(n > 0);
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n, 1, 1), 
+						   X(mktensor_0d)(), 
+						   buf, buf, R2HC));
+     X(ifree)(buf);
+     if (!cld)
+          return (plan *)0;
+
+     pln = MKPLAN_RDFT(P, &padt, apply);
+
+     pln->n = n;
+     pln->is = p->sz->dims[0].is;
+     pln->os = p->sz->dims[0].os;
+     pln->cld = cld;
+     pln->td = 0;
+
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     
+     X(ops_zero)(&ops);
+     ops.other = 8 + (n-1)/2 * 11 + (1 - n % 2) * 5;
+     ops.add = 2 + (n-1)/2 * 5;
+     ops.mul = (n-1)/2 * 3 + (1 - n % 2) * 1;
+
+     X(ops_zero)(&pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(redft00e_r2hc_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/reodft/reoconf.c b/src/fftw3/reodft/reoconf.c
new file mode 100644
index 0000000..1cd41b6
--- /dev/null
+++ b/src/fftw3/reodft/reoconf.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: reoconf.c,v 1.1 2008/10/17 06:13:18 scuri Exp $ */
+
+#include "reodft.h"
+
+static const solvtab s =
+{
+     /* SOLVTAB(X(redft00e_r2hc_register)),
+	SOLVTAB(X(rodft00e_r2hc_register)), */
+     SOLVTAB(X(redft00e_r2hc_pad_register)),
+     SOLVTAB(X(rodft00e_r2hc_pad_register)),
+     SOLVTAB(X(reodft010e_r2hc_register)),
+     /* SOLVTAB(X(reodft11e_r2hc_register)), */
+     SOLVTAB(X(reodft11e_radix2_r2hc_register)),
+     SOLVTAB(X(reodft11e_r2hc_odd_register)),
+
+     SOLVTAB_END
+};
+
+void X(reodft_conf_standard)(planner *p)
+{
+     X(solvtab_exec)(s, p);
+}
diff --git a/src/fftw3/reodft/reodft.h b/src/fftw3/reodft/reodft.h
new file mode 100644
index 0000000..8c67144
--- /dev/null
+++ b/src/fftw3/reodft/reodft.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#ifndef __REODFT_H__
+#define __REODFT_H__
+
+#include "ifftw.h"
+#include "rdft.h"
+
+#define REODFT_KINDP(k) ((k) >= REDFT00 && (k) <= RODFT11)
+
+void X(redft00e_r2hc_register)(planner *p);
+void X(redft00e_r2hc_pad_register)(planner *p);
+void X(rodft00e_r2hc_register)(planner *p);
+void X(rodft00e_r2hc_pad_register)(planner *p);
+void X(reodft010e_r2hc_register)(planner *p);
+void X(reodft11e_r2hc_register)(planner *p);
+void X(reodft11e_radix2_r2hc_register)(planner *p);
+void X(reodft11e_r2hc_odd_register)(planner *p);
+
+/* configurations */
+void X(reodft_conf_standard)(planner *p);
+
+#endif /* __REODFT_H__ */
diff --git a/src/fftw3/reodft/reodft010e-r2hc.c b/src/fftw3/reodft/reodft010e-r2hc.c
new file mode 100644
index 0000000..ace14de
--- /dev/null
+++ b/src/fftw3/reodft/reodft010e-r2hc.c
@@ -0,0 +1,409 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: reodft010e-r2hc.c,v 1.1 2008/10/17 06:13:18 scuri Exp $ */
+
+/* Do an R{E,O}DFT{01,10} problem via an R2HC problem, with some
+   pre/post-processing ala FFTPACK. */
+
+#include "reodft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cld;
+     twid *td;
+     int is, os;
+     int n;
+     int vl;
+     int ivs, ovs;
+     rdft_kind kind;
+} P;
+
+/* A real-even-01 DFT operates logically on a size-4N array:
+                   I 0 -r(I*) -I 0 r(I*),
+   where r denotes reversal and * denotes deletion of the 0th element.
+   To compute the transform of this, we imagine performing a radix-4
+   (real-input) DIF step, which turns the size-4N DFT into 4 size-N
+   (contiguous) DFTs, two of which are zero and two of which are
+   conjugates.  The non-redundant size-N DFT has halfcomplex input, so
+   we can do it with a size-N hc2r transform.  (In order to share
+   plans with the re10 (inverse) transform, however, we use the DHT
+   trick to re-express the hc2r problem as r2hc.  This has little cost
+   since we are already pre- and post-processing the data in {i,n-i}
+   order.)  Finally, we have to write out the data in the correct
+   order...the two size-N redundant (conjugate) hc2r DFTs correspond
+   to the even and odd outputs in O (i.e. the usual interleaved output
+   of DIF transforms); since this data has even symmetry, we only
+   write the first half of it.
+
+   The real-even-10 DFT is just the reverse of these steps, i.e. a
+   radix-4 DIT transform.  There, however, we just use the r2hc
+   transform naturally without resorting to the DHT trick.
+
+   A real-odd-01 DFT is very similar, except that the input is
+   0 I (rI)* 0 -I -(rI)*.  This format, however, can be transformed
+   into precisely the real-even-01 format above by sending I -> rI
+   and shifting the array by N.  The former swap is just another
+   transformation on the input during preprocessing; the latter
+   multiplies the even/odd outputs by i/-i, which combines with
+   the factor of -i (to take the imaginary part) to simply flip
+   the sign of the odd outputs.  Vice-versa for real-odd-10.
+
+   The FFTPACK source code was very helpful in working this out.
+   (They do unnecessary passes over the array, though.)
+
+   Note that Numerical Recipes suggests a different algorithm that
+   requires more operations and uses trig. functions for both the pre-
+   and post-processing passes.
+*/
+
+static void apply_re01(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *W = ego->td->W;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  buf[0] = I[0];
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b, apb, amb, wa, wb;
+	       a = I[is * i];
+	       b = I[is * (n - i)];
+	       apb = a + b;
+	       amb = a - b;
+	       wa = W[2*i];
+	       wb = W[2*i + 1];
+	       buf[i] = wa * amb + wb * apb; 
+	       buf[n - i] = wa * apb - wb * amb; 
+	  }
+	  if (i == n - i) {
+	       buf[i] = K(2.0) * I[is * i] * W[2*i];
+	  }
+	  
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  O[0] = buf[0];
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b;
+	       int k;
+	       a = buf[i];
+	       b = buf[n - i];
+	       k = i + i;
+	       O[os * (k - 1)] = a - b;
+	       O[os * k] = a + b;
+	  }
+	  if (i == n - i) {
+	       O[os * (n - 1)] = buf[i];
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+/* ro01 is same as re01, but with i <-> n - 1 - i in the input and
+   the sign of the odd output elements flipped. */
+static void apply_ro01(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *W = ego->td->W;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  buf[0] = I[is * (n - 1)];
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b, apb, amb, wa, wb;
+	       a = I[is * (n - 1 - i)];
+	       b = I[is * (i - 1)];
+	       apb = a + b;
+	       amb = a - b;
+	       wa = W[2*i];
+	       wb = W[2*i+1];
+	       buf[i] = wa * amb + wb * apb; 
+	       buf[n - i] = wa * apb - wb * amb; 
+	  }
+	  if (i == n - i) {
+	       buf[i] = K(2.0) * I[is * (i - 1)] * W[2*i];
+	  }
+	  
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  O[0] = buf[0];
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b;
+	       int k;
+	       a = buf[i];
+	       b = buf[n - i];
+	       k = i + i;
+	       O[os * (k - 1)] = b - a;
+	       O[os * k] = a + b;
+	  }
+	  if (i == n - i) {
+	       O[os * (n - 1)] = -buf[i];
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+static void apply_re10(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *W = ego->td->W;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  buf[0] = I[0];
+	  for (i = 1; i < n - i; ++i) {
+	       E u, v;
+	       int k = i + i;
+	       u = I[is * (k - 1)];
+	       v = I[is * k];
+	       buf[n - i] = u;
+	       buf[i] = v;
+	  }
+	  if (i == n - i) {
+	       buf[i] = I[is * (n - 1)];
+	  }
+	  
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  O[0] = K(2.0) * buf[0];
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b, wa, wb;
+	       a = K(2.0) * buf[i];
+	       b = K(2.0) * buf[n - i];
+	       wa = W[2*i];
+	       wb = W[2*i + 1];
+	       O[os * i] = wa * a + wb * b;
+	       O[os * (n - i)] = wb * a - wa * b;
+	  }
+	  if (i == n - i) {
+	       O[os * i] = K(2.0) * buf[i] * W[2*i];
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+/* ro10 is same as re10, but with i <-> n - 1 - i in the output and
+   the sign of the odd input elements flipped. */
+static void apply_ro10(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *W = ego->td->W;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  buf[0] = I[0];
+	  for (i = 1; i < n - i; ++i) {
+	       E u, v;
+	       int k = i + i;
+	       u = -I[is * (k - 1)];
+	       v = I[is * k];
+	       buf[n - i] = u;
+	       buf[i] = v;
+	  }
+	  if (i == n - i) {
+	       buf[i] = -I[is * (n - 1)];
+	  }
+	  
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  O[os * (n - 1)] = K(2.0) * buf[0];
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b, wa, wb;
+	       a = K(2.0) * buf[i];
+	       b = K(2.0) * buf[n - i];
+	       wa = W[2*i];
+	       wb = W[2*i + 1];
+	       O[os * (n - 1 - i)] = wa * a + wb * b;
+	       O[os * (i - 1)] = wb * a - wa * b;
+	  }
+	  if (i == n - i) {
+	       O[os * (i - 1)] = K(2.0) * buf[i] * W[2*i];
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     static const tw_instr reodft010e_tw[] = {
+          { TW_COS, 0, 1 },
+          { TW_SIN, 0, 1 },
+          { TW_NEXT, 1, 0 }
+     };
+
+     AWAKE(ego->cld, flg);
+
+     X(twiddle_awake)(flg, &ego->td, reodft010e_tw, 4*ego->n, 1, ego->n/2+1);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(%se-r2hc-%d%v%(%p%))",
+	      X(rdft_kind_str)(ego->kind), ego->n, ego->vl, ego->cld);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk <= 1
+		  && (p->kind[0] == REDFT01 || p->kind[0] == REDFT10
+		      || p->kind[0] == RODFT01 || p->kind[0] == RODFT10)
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p, const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0(ego, p));
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     P *pln;
+     const problem_rdft *p;
+     plan *cld;
+     R *buf;
+     int n;
+     opcnt ops;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+          return (plan *)0;
+
+     p = (const problem_rdft *) p_;
+
+     n = p->sz->dims[0].n;
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n, 1, 1),
+                                                   X(mktensor_0d)(),
+                                                   buf, buf, R2HC));
+     X(ifree)(buf);
+     if (!cld)
+          return (plan *)0;
+
+     switch (p->kind[0]) {
+	 case REDFT01: pln = MKPLAN_RDFT(P, &padt, apply_re01); break;
+	 case REDFT10: pln = MKPLAN_RDFT(P, &padt, apply_re10); break;
+	 case RODFT01: pln = MKPLAN_RDFT(P, &padt, apply_ro01); break;
+	 case RODFT10: pln = MKPLAN_RDFT(P, &padt, apply_ro10); break;
+	 default: A(0); return (plan*)0;
+     }
+
+     pln->n = n;
+     pln->is = p->sz->dims[0].is;
+     pln->os = p->sz->dims[0].os;
+     pln->cld = cld;
+     pln->td = 0;
+     pln->kind = p->kind[0];
+     
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     
+     X(ops_zero)(&ops);
+     ops.other = 4 + (n-1)/2 * 10 + (1 - n % 2) * 5;
+     if (p->kind[0] == REDFT01 || p->kind[0] == RODFT01) {
+	  ops.add = (n-1)/2 * 6;
+	  ops.mul = (n-1)/2 * 4 + (1 - n % 2) * 2;
+     }
+     else { /* 10 transforms */
+	  ops.add = (n-1)/2 * 2;
+	  ops.mul = 1 + (n-1)/2 * 6 + (1 - n % 2) * 2;
+     }
+     
+     X(ops_zero)(&pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(reodft010e_r2hc_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/reodft/reodft11e-r2hc-odd.c b/src/fftw3/reodft/reodft11e-r2hc-odd.c
new file mode 100644
index 0000000..471f7ca
--- /dev/null
+++ b/src/fftw3/reodft/reodft11e-r2hc-odd.c
@@ -0,0 +1,304 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: reodft11e-r2hc-odd.c,v 1.1 2008/10/17 06:13:18 scuri Exp $ */
+
+/* Do an R{E,O}DFT11 problem via an R2HC problem of the same *odd* size,
+   with some permutations and post-processing, as described in:
+
+     S. C. Chan and K. L. Ho, "Fast algorithms for computing the
+     discrete cosine transform," IEEE Trans. Circuits Systems II:
+     Analog & Digital Sig. Proc. 39 (3), 185--190 (1992).
+
+   (For even sizes, see reodft11e-radix2.c.)  
+
+   This algorithm is related to the 8 x n prime-factor-algorithm (PFA)
+   decomposition of the size 8n "logical" DFT corresponding to the
+   R{EO}DFT11.
+
+   Aside from very confusing notation (several symbols are redefined
+   from one line to the next), be aware that this paper has some
+   errors.  In particular, the signs are wrong in Eqs. (34-35).  Also,
+   Eqs. (36-37) should be simply C(k) = C(2k + 1 mod N), and similarly
+   for S (or, equivalently, the second cases should have 2*N - 2*k - 1
+   instead of N - k - 1).  Note also that in their definition of the
+   DFT, similarly to FFTW's, the exponent's sign is -1, but they
+   forgot to correspondingly multiply S (the sine terms) by -1.
+*/
+
+#include "reodft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cld;
+     int is, os;
+     int n;
+     int vl;
+     int ivs, ovs;
+     rdft_kind kind;
+} P;
+
+static DK(SQRT2, +1.4142135623730950488016887242096980785696718753769);
+
+#define SGN_SET(x, i) ((i) % 2 ? -(x) : (x))
+
+static void apply_re11(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n, n2 = n/2;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  {
+	       int m;
+	       for (i = 0, m = n2; m < n; ++i, m += 4)
+		    buf[i] = I[is * m];
+	       for (; m < 2 * n; ++i, m += 4)
+		    buf[i] = -I[is * (2*n - m - 1)];
+	       for (; m < 3 * n; ++i, m += 4)
+		    buf[i] = -I[is * (m - 2*n)];
+	       for (; m < 4 * n; ++i, m += 4)
+		    buf[i] = I[is * (4*n - m - 1)];
+	       m -= 4 * n;
+	       for (; i < n; ++i, m += 4)
+		    buf[i] = I[is * m];
+	  }
+
+	  { /* child plan: R2HC of size n */
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  /* FIXME: strength-reduce loop by 4 to eliminate ugly sgn_set? */
+	  for (i = 0; i + i + 1 < n2; ++i) {
+	       int k = i + i + 1;
+	       E c1, s1;
+	       E c2, s2;
+	       c1 = buf[k];
+	       c2 = buf[k + 1];
+	       s2 = buf[n - (k + 1)];
+	       s1 = buf[n - k];
+	       
+	       O[os * i] = SQRT2 * (SGN_SET(c1, (i+1)/2) +
+				    SGN_SET(s1, i/2));
+	       O[os * (n - (i+1))] = SQRT2 * (SGN_SET(c1, (n-i)/2) -
+					      SGN_SET(s1, (n-(i+1))/2));
+	       
+	       O[os * (n2 - (i+1))] = SQRT2 * (SGN_SET(c2, (n2-i)/2) -
+					       SGN_SET(s2, (n2-(i+1))/2));
+	       O[os * (n2 + (i+1))] = SQRT2 * (SGN_SET(c2, (n2+i+2)/2) +
+					       SGN_SET(s2, (n2+(i+1))/2));
+	  }
+	  if (i + i + 1 == n2) {
+	       E c, s;
+	       c = buf[n2];
+	       s = buf[n - n2];
+	       O[os * i] = SQRT2 * (SGN_SET(c, (i+1)/2) +
+				    SGN_SET(s, i/2));
+	       O[os * (n - (i+1))] = SQRT2 * (SGN_SET(c, (i+2)/2) +
+					      SGN_SET(s, (i+1)/2));
+	  }
+	  O[os * n2] = SQRT2 * SGN_SET(buf[0], (n2+1)/2);
+     }
+
+     X(ifree)(buf);
+}
+
+/* like for rodft01, rodft11 is obtained from redft11 by
+   reversing the input and flipping the sign of every other output. */
+static void apply_ro11(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n, n2 = n/2;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  {
+	       int m;
+	       for (i = 0, m = n2; m < n; ++i, m += 4)
+		    buf[i] = I[is * (n - 1 - m)];
+	       for (; m < 2 * n; ++i, m += 4)
+		    buf[i] = -I[is * (m - n)];
+	       for (; m < 3 * n; ++i, m += 4)
+		    buf[i] = -I[is * (3*n - 1 - m)];
+	       for (; m < 4 * n; ++i, m += 4)
+		    buf[i] = I[is * (m - 3*n)];
+	       m -= 4 * n;
+	       for (; i < n; ++i, m += 4)
+		    buf[i] = I[is * (n - 1 - m)];
+	  }
+
+	  { /* child plan: R2HC of size n */
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  /* FIXME: strength-reduce loop by 4 to eliminate ugly sgn_set? */
+	  for (i = 0; i + i + 1 < n2; ++i) {
+	       int k = i + i + 1;
+	       int j;
+	       E c1, s1;
+	       E c2, s2;
+	       c1 = buf[k];
+	       c2 = buf[k + 1];
+	       s2 = buf[n - (k + 1)];
+	       s1 = buf[n - k];
+	       
+	       O[os * i] = SQRT2 * (SGN_SET(c1, (i+1)/2 + i) +
+				    SGN_SET(s1, i/2 + i));
+	       O[os * (n - (i+1))] = SQRT2 * (SGN_SET(c1, (n-i)/2 + i) -
+					      SGN_SET(s1, (n-(i+1))/2 + i));
+	       
+	       j = n2 - (i+1);
+	       O[os * j] = SQRT2 * (SGN_SET(c2, (n2-i)/2 + j) -
+				    SGN_SET(s2, (n2-(i+1))/2 + j));
+	       O[os * (n2 + (i+1))] = SQRT2 * (SGN_SET(c2, (n2+i+2)/2 + j) +
+					       SGN_SET(s2, (n2+(i+1))/2 + j));
+	  }
+	  if (i + i + 1 == n2) {
+	       E c, s;
+	       c = buf[n2];
+	       s = buf[n - n2];
+	       O[os * i] = SQRT2 * (SGN_SET(c, (i+1)/2 + i) +
+				    SGN_SET(s, i/2 + i));
+	       O[os * (n - (i+1))] = SQRT2 * (SGN_SET(c, (i+2)/2 + i) +
+					      SGN_SET(s, (i+1)/2 + i));
+	  }
+	  O[os * n2] = SQRT2 * SGN_SET(buf[0], (n2+1)/2 + n2);
+     }
+
+     X(ifree)(buf);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cld, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(%se-r2hc-odd-%d%v%(%p%))",
+	      X(rdft_kind_str)(ego->kind), ego->n, ego->vl, ego->cld);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk <= 1
+		  && p->sz->dims[0].n % 2 == 1
+		  && (p->kind[0] == REDFT11 || p->kind[0] == RODFT11)
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p, const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0(ego, p));
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     P *pln;
+     const problem_rdft *p;
+     plan *cld;
+     R *buf;
+     int n;
+     opcnt ops;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+          return (plan *)0;
+
+     p = (const problem_rdft *) p_;
+
+     n = p->sz->dims[0].n;
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n, 1, 1),
+                                                   X(mktensor_0d)(),
+                                                   buf, buf, R2HC));
+     X(ifree)(buf);
+     if (!cld)
+          return (plan *)0;
+
+     pln = MKPLAN_RDFT(P, &padt, p->kind[0]==REDFT11 ? apply_re11:apply_ro11);
+     pln->n = n;
+     pln->is = p->sz->dims[0].is;
+     pln->os = p->sz->dims[0].os;
+     pln->cld = cld;
+     pln->kind = p->kind[0];
+     
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     
+     X(ops_zero)(&ops);
+     ops.add = n - 1;
+     ops.mul = n;
+     ops.other = 4*n;
+
+     X(ops_zero)(&pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(reodft11e_r2hc_odd_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/reodft/reodft11e-r2hc.c b/src/fftw3/reodft/reodft11e-r2hc.c
new file mode 100644
index 0000000..d4366e3
--- /dev/null
+++ b/src/fftw3/reodft/reodft11e-r2hc.c
@@ -0,0 +1,295 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: reodft11e-r2hc.c,v 1.1 2008/10/17 06:13:18 scuri Exp $ */
+
+/* Do an R{E,O}DFT11 problem via an R2HC problem, with some
+   pre/post-processing ala FFTPACK.  Use a trick from: 
+
+     S. C. Chan and K. L. Ho, "Direct methods for computing discrete
+     sinusoidal transforms," IEE Proceedings F 137 (6), 433--442 (1990).
+
+   to re-express as an REDFT01 (DCT-III) problem.
+
+   NOTE: We no longer use this algorithm, because it turns out to suffer
+   a catastrophic loss of accuracy for certain inputs, apparently because
+   its post-processing multiplies the output by a cosine.  Near the zero
+   of the cosine, the REDFT01 must produce a near-singular output.
+*/
+
+#include "reodft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cld;
+     twid *td, *td2;
+     int is, os;
+     int n;
+     int vl;
+     int ivs, ovs;
+     rdft_kind kind;
+} P;
+
+static void apply_re11(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *W;
+     R *buf;
+     E cur;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  /* I wish that this didn't require an extra pass. */
+	  /* FIXME: use recursive/cascade summation for better stability? */
+	  buf[n - 1] = cur = K(2.0) * I[is * (n - 1)];
+	  for (i = n - 1; i > 0; --i) {
+	       E curnew;
+	       buf[(i - 1)] = curnew = K(2.0) * I[is * (i - 1)] - cur;
+	       cur = curnew;
+	  }
+	  
+	  W = ego->td->W;
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b, apb, amb, wa, wb;
+	       a = buf[i];
+	       b = buf[n - i];
+	       apb = a + b;
+	       amb = a - b;
+	       wa = W[2*i];
+	       wb = W[2*i + 1];
+	       buf[i] = wa * amb + wb * apb; 
+	       buf[n - i] = wa * apb - wb * amb; 
+	  }
+	  if (i == n - i) {
+	       buf[i] = K(2.0) * buf[i] * W[2*i];
+	  }
+	  
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  W = ego->td2->W;
+	  O[0] = W[0] * buf[0];
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b;
+	       int k;
+	       a = buf[i];
+	       b = buf[n - i];
+	       k = i + i;
+	       O[os * (k - 1)] = W[k - 1] * (a - b);
+	       O[os * k] = W[k] * (a + b);
+	  }
+	  if (i == n - i) {
+	       O[os * (n - 1)] = W[n - 1] * buf[i];
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+/* like for rodft01, rodft11 is obtained from redft11 by
+   reversing the input and flipping the sign of every other output. */
+static void apply_ro11(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *W;
+     R *buf;
+     E cur;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  /* I wish that this didn't require an extra pass. */
+	  /* FIXME: use recursive/cascade summation for better stability? */
+	  buf[n - 1] = cur = K(2.0) * I[0];
+	  for (i = n - 1; i > 0; --i) {
+	       E curnew;
+	       buf[(i - 1)] = curnew = K(2.0) * I[is * (n - i)] - cur;
+	       cur = curnew;
+	  }
+	  
+	  W = ego->td->W;
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b, apb, amb, wa, wb;
+	       a = buf[i];
+	       b = buf[n - i];
+	       apb = a + b;
+	       amb = a - b;
+	       wa = W[2*i];
+	       wb = W[2*i + 1];
+	       buf[i] = wa * amb + wb * apb; 
+	       buf[n - i] = wa * apb - wb * amb; 
+	  }
+	  if (i == n - i) {
+	       buf[i] = K(2.0) * buf[i] * W[2*i];
+	  }
+	  
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  W = ego->td2->W;
+	  O[0] = W[0] * buf[0];
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b;
+	       int k;
+	       a = buf[i];
+	       b = buf[n - i];
+	       k = i + i;
+	       O[os * (k - 1)] = W[k - 1] * (b - a);
+	       O[os * k] = W[k] * (a + b);
+	  }
+	  if (i == n - i) {
+	       O[os * (n - 1)] = -W[n - 1] * buf[i];
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     static const tw_instr reodft010e_tw[] = {
+          { TW_COS, 0, 1 },
+          { TW_SIN, 0, 1 },
+          { TW_NEXT, 1, 0 }
+     };
+     static const tw_instr reodft11e_tw[] = {
+          { TW_COS, 1, 1 },
+          { TW_NEXT, 2, 0 }
+     };
+
+     AWAKE(ego->cld, flg);
+
+     X(twiddle_awake)(flg, &ego->td, reodft010e_tw, 4*ego->n, 1, ego->n/2+1);
+     X(twiddle_awake)(flg, &ego->td2, reodft11e_tw, 8*ego->n, 1, ego->n * 2);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(%se-r2hc-%d%v%(%p%))",
+	      X(rdft_kind_str)(ego->kind), ego->n, ego->vl, ego->cld);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk <= 1
+		  && (p->kind[0] == REDFT11 || p->kind[0] == RODFT11)
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p, const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0(ego, p));
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     P *pln;
+     const problem_rdft *p;
+     plan *cld;
+     R *buf;
+     int n;
+     opcnt ops;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+          return (plan *)0;
+
+     p = (const problem_rdft *) p_;
+
+     n = p->sz->dims[0].n;
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n, 1, 1),
+                                                   X(mktensor_0d)(),
+                                                   buf, buf, R2HC));
+     X(ifree)(buf);
+     if (!cld)
+          return (plan *)0;
+
+     pln = MKPLAN_RDFT(P, &padt, p->kind[0]==REDFT11 ? apply_re11:apply_ro11);
+     pln->n = n;
+     pln->is = p->sz->dims[0].is;
+     pln->os = p->sz->dims[0].os;
+     pln->cld = cld;
+     pln->td = pln->td2 = 0;
+     pln->kind = p->kind[0];
+     
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     
+     X(ops_zero)(&ops);
+     ops.other = 5 + (n-1) * 2 + (n-1)/2 * 12 + (1 - n % 2) * 6;
+     ops.add = (n - 1) * 1 + (n-1)/2 * 6;
+     ops.mul = 2 + (n-1) * 1 + (n-1)/2 * 6 + (1 - n % 2) * 3;
+
+     X(ops_zero)(&pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(reodft11e_r2hc_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/reodft/reodft11e-radix2.c b/src/fftw3/reodft/reodft11e-radix2.c
new file mode 100644
index 0000000..674f7b4
--- /dev/null
+++ b/src/fftw3/reodft/reodft11e-radix2.c
@@ -0,0 +1,515 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: reodft11e-radix2.c,v 1.1 2008/10/17 06:13:18 scuri Exp $ */
+
+/* Do an R{E,O}DFT11 problem of *even* size by a pair of R2HC problems
+   of half the size, plus some pre/post-processing.  Use a trick from:
+
+   Zhongde Wang, "On computing the discrete Fourier and cosine transforms,"
+   IEEE Trans. Acoust. Speech Sig. Proc. ASSP-33 (4), 1341--1344 (1985).
+
+   to re-express as a pair of half-size REDFT01 (DCT-III) problems.  Our
+   implementation looks quite a bit different from the algorithm described
+   in the paper because we combined the paper's pre/post-processing with
+   the pre/post-processing used to turn REDFT01 into R2HC.  (Also, the
+   paper uses a DCT/DST pair, but we turn the DST into a DCT via the
+   usual reordering/sign-flip trick.  We additionally combined a couple
+   of the matrices/transformations of the paper into a single pass.)
+
+   NOTE: We originally used a simpler method by S. C. Chan and K. L. Ho
+   that turned out to have numerical problems; see reodft11e-r2hc.c.
+
+   (For odd sizes, see reodft11e-r2hc-odd.c.)
+*/
+
+#include "reodft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cld;
+     twid *td, *td2;
+     int is, os;
+     int n;
+     int vl;
+     int ivs, ovs;
+     rdft_kind kind;
+} P;
+
+static void apply_re11(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n, n2 = n/2;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *W = ego->td->W;
+     R *W2;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  buf[0] = K(2.0) * I[0];
+	  buf[n2] = K(2.0) * I[is * (n - 1)];
+	  for (i = 1; i + i < n2; ++i) {
+	       int k = i + i;
+	       E a, b, a2, b2;
+	       {
+		    E u, v;
+		    u = I[is * (k - 1)];
+		    v = I[is * k];
+		    a = u + v;
+		    b2 = u - v;
+	       }
+	       {
+		    E u, v;
+		    u = I[is * (n - k - 1)];
+		    v = I[is * (n - k)];
+		    b = u + v;
+		    a2 = u - v;
+	       }
+	       {
+		    E wa, wb;
+		    wa = W[2*i];
+		    wb = W[2*i + 1];
+		    {
+			 E apb, amb;
+			 apb = a + b;
+			 amb = a - b;
+			 buf[i] = wa * amb + wb * apb; 
+			 buf[n2 - i] = wa * apb - wb * amb; 
+		    }
+		    {
+			 E apb, amb;
+			 apb = a2 + b2;
+			 amb = a2 - b2;
+			 buf[n2 + i] = wa * amb + wb * apb; 
+			 buf[n - i] = wa * apb - wb * amb; 
+		    }
+	       }
+	  }
+	  if (i + i == n2) {
+	       E u, v;
+	       u = I[is * (n2 - 1)];
+	       v = I[is * n2];
+	       buf[i] = K(2.0) * (u + v) * W[2*i];
+	       buf[n - i] = K(2.0) * (u - v) * W[2*i];
+	  }
+
+
+	  /* child plan: two r2hc's of size n/2 */
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  W2 = ego->td2->W;
+	  { /* i == 0 case */
+	       E wa, wb;
+	       E a, b;
+	       wa = W2[0]; /* cos */
+	       wb = W2[1]; /* sin */
+	       a = buf[0];
+	       b = buf[n2];
+	       O[0] = wa * a + wb * b;
+	       O[os * (n - 1)] = wb * a - wa * b;
+	  }
+	  W2 += 2;
+	  for (i = 1; i + i < n2; ++i, W2 += 2) {
+	       int k;
+	       E u, v, u2, v2;
+	       u = buf[i];
+	       v = buf[n2 - i];
+	       u2 = buf[n2 + i];
+	       v2 = buf[n - i];
+	       k = (i + i) - 1;
+	       {
+                    E wa, wb;
+                    E a, b;
+                    wa = W2[0]; /* cos */
+                    wb = W2[1]; /* sin */
+                    a = u - v;
+                    b = v2 - u2;
+                    O[os * k] = wa * a + wb * b;
+                    O[os * (n - 1 - k)] = wb * a - wa * b;
+               }
+	       ++k;
+	       W2 += 2;
+	       {
+		    E wa, wb;
+		    E a, b;
+		    wa = W2[0]; /* cos */
+		    wb = W2[1]; /* sin */
+		    a = u + v;
+		    b = u2 + v2;
+		    O[os * k] = wa * a + wb * b;
+		    O[os * (n - 1 - k)] = wb * a - wa * b;
+	       }
+	  }
+	  if (i + i == n2) {
+	       int k = (i + i) - 1;
+	       E wa, wb;
+	       E a, b;
+	       wa = W2[0]; /* cos */
+	       wb = W2[1]; /* sin */
+	       a = buf[i];
+	       b = buf[n2 + i];
+	       O[os * k] = wa * a - wb * b;
+	       O[os * (n - 1 - k)] = wb * a + wa * b;
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+#if 0
+
+/* This version of apply_re11 uses REDFT01 child plans, more similar
+   to the original paper by Z. Wang.  We keep it around for reference
+   (it is simpler) and because it may become more efficient if we
+   ever implement REDFT01 codelets. */
+
+static void apply_re11(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *W;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  buf[0] = K(2.0) * I[0];
+	  buf[n/2] = K(2.0) * I[is * (n - 1)];
+	  for (i = 1; i + i < n; ++i) {
+	       int k = i + i;
+	       E a, b;
+	       a = I[is * (k - 1)];
+	       b = I[is * k];
+	       buf[i] = a + b;
+	       buf[n - i] = a - b;
+	  }
+
+	  /* child plan: two redft01's (DCT-III) */
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  W = ego->td2->W;
+	  for (i = 0; i + 1 < n/2; ++i, W += 2) {
+	       {
+		    E wa, wb;
+		    E a, b;
+		    wa = W[0]; /* cos */
+		    wb = W[1]; /* sin */
+		    a = buf[i];
+		    b = buf[n/2 + i];
+		    O[os * i] = wa * a + wb * b;
+		    O[os * (n - 1 - i)] = wb * a - wa * b;
+	       }
+	       ++i;
+	       W += 2;
+	       {
+                    E wa, wb;
+                    E a, b;
+                    wa = W[0]; /* cos */
+                    wb = W[1]; /* sin */
+                    a = buf[i];
+                    b = buf[n/2 + i];
+                    O[os * i] = wa * a - wb * b;
+                    O[os * (n - 1 - i)] = wb * a + wa * b;
+               }
+	  }
+	  if (i < n/2) {
+	       E wa, wb;
+	       E a, b;
+	       wa = W[0]; /* cos */
+	       wb = W[1]; /* sin */
+	       a = buf[i];
+	       b = buf[n/2 + i];
+	       O[os * i] = wa * a + wb * b;
+	       O[os * (n - 1 - i)] = wb * a - wa * b;
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+#endif /* 0 */
+
+/* like for rodft01, rodft11 is obtained from redft11 by
+   reversing the input and flipping the sign of every other output. */
+static void apply_ro11(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n, n2 = n/2;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *W = ego->td->W;
+     R *W2;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  buf[0] = K(2.0) * I[is * (n - 1)];
+	  buf[n2] = K(2.0) * I[0];
+	  for (i = 1; i + i < n2; ++i) {
+	       int k = i + i;
+	       E a, b, a2, b2;
+	       {
+		    E u, v;
+		    u = I[is * (n - k)];
+		    v = I[is * (n - 1 - k)];
+		    a = u + v;
+		    b2 = u - v;
+	       }
+	       {
+		    E u, v;
+		    u = I[is * (k)];
+		    v = I[is * (k - 1)];
+		    b = u + v;
+		    a2 = u - v;
+	       }
+	       {
+		    E wa, wb;
+		    wa = W[2*i];
+		    wb = W[2*i + 1];
+		    {
+			 E apb, amb;
+			 apb = a + b;
+			 amb = a - b;
+			 buf[i] = wa * amb + wb * apb; 
+			 buf[n2 - i] = wa * apb - wb * amb; 
+		    }
+		    {
+			 E apb, amb;
+			 apb = a2 + b2;
+			 amb = a2 - b2;
+			 buf[n2 + i] = wa * amb + wb * apb; 
+			 buf[n - i] = wa * apb - wb * amb; 
+		    }
+	       }
+	  }
+	  if (i + i == n2) {
+	       E u, v;
+	       u = I[is * n2];
+	       v = I[is * (n2 - 1)];
+	       buf[i] = K(2.0) * (u + v) * W[2*i];
+	       buf[n - i] = K(2.0) * (u - v) * W[2*i];
+	  }
+
+
+	  /* child plan: two r2hc's of size n/2 */
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  W2 = ego->td2->W;
+	  { /* i == 0 case */
+	       E wa, wb;
+	       E a, b;
+	       wa = W2[0]; /* cos */
+	       wb = W2[1]; /* sin */
+	       a = buf[0];
+	       b = buf[n2];
+	       O[0] = wa * a + wb * b;
+	       O[os * (n - 1)] = wa * b - wb * a;
+	  }
+	  W2 += 2;
+	  for (i = 1; i + i < n2; ++i, W2 += 2) {
+	       int k;
+	       E u, v, u2, v2;
+	       u = buf[i];
+	       v = buf[n2 - i];
+	       u2 = buf[n2 + i];
+	       v2 = buf[n - i];
+	       k = (i + i) - 1;
+	       {
+                    E wa, wb;
+                    E a, b;
+                    wa = W2[0]; /* cos */
+                    wb = W2[1]; /* sin */
+                    a = v - u;
+                    b = u2 - v2;
+                    O[os * k] = wa * a + wb * b;
+                    O[os * (n - 1 - k)] = wa * b - wb * a;
+               }
+	       ++k;
+	       W2 += 2;
+	       {
+		    E wa, wb;
+		    E a, b;
+		    wa = W2[0]; /* cos */
+		    wb = W2[1]; /* sin */
+		    a = u + v;
+		    b = u2 + v2;
+		    O[os * k] = wa * a + wb * b;
+		    O[os * (n - 1 - k)] = wa * b - wb * a;
+	       }
+	  }
+	  if (i + i == n2) {
+	       int k = (i + i) - 1;
+	       E wa, wb;
+	       E a, b;
+	       wa = W2[0]; /* cos */
+	       wb = W2[1]; /* sin */
+	       a = buf[i];
+	       b = buf[n2 + i];
+	       O[os * k] = wb * b - wa * a;
+	       O[os * (n - 1 - k)] = wa * b + wb * a;
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     static const tw_instr reodft010e_tw[] = {
+          { TW_COS, 0, 1 },
+          { TW_SIN, 0, 1 },
+          { TW_NEXT, 1, 0 }
+     };
+     static const tw_instr reodft11e_tw[] = {
+          { TW_COS, 1, 1 },
+          { TW_SIN, 1, 1 },
+          { TW_NEXT, 2, 0 }
+     };
+
+     AWAKE(ego->cld, flg);
+
+     X(twiddle_awake)(flg, &ego->td, reodft010e_tw, 2*ego->n, 1, ego->n/4+1);
+     X(twiddle_awake)(flg, &ego->td2, reodft11e_tw, 8*ego->n, 1, ego->n);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(%se-radix2-r2hc-%d%v%(%p%))",
+	      X(rdft_kind_str)(ego->kind), ego->n, ego->vl, ego->cld);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk <= 1
+		  && p->sz->dims[0].n % 2 == 0
+		  && (p->kind[0] == REDFT11 || p->kind[0] == RODFT11)
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p, const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0(ego, p));
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     P *pln;
+     const problem_rdft *p;
+     plan *cld;
+     R *buf;
+     int n;
+     opcnt ops;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+          return (plan *)0;
+
+     p = (const problem_rdft *) p_;
+
+     n = p->sz->dims[0].n;
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n/2, 1, 1),
+                                                   X(mktensor_1d)(2, n/2, n/2),
+                                                   buf, buf, R2HC));
+     X(ifree)(buf);
+     if (!cld)
+          return (plan *)0;
+
+     pln = MKPLAN_RDFT(P, &padt, p->kind[0]==REDFT11 ? apply_re11:apply_ro11);
+     pln->n = n;
+     pln->is = p->sz->dims[0].is;
+     pln->os = p->sz->dims[0].os;
+     pln->cld = cld;
+     pln->td = pln->td2 = 0;
+     pln->kind = p->kind[0];
+     
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     
+     X(ops_zero)(&ops);
+     ops.add = 2 + (n/2 - 1)/2 * 20;
+     ops.mul = 6 + (n/2 - 1)/2 * 16;
+     ops.other = 4*n + 2 + (n/2 - 1)/2 * 6;
+     if ((n/2) % 2 == 0) {
+	  ops.add += 4;
+	  ops.mul += 8;
+	  ops.other += 4;
+     }
+
+     X(ops_zero)(&pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(reodft11e_radix2_r2hc_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/reodft/rodft00e-r2hc-pad.c b/src/fftw3/reodft/rodft00e-r2hc-pad.c
new file mode 100644
index 0000000..0b48585
--- /dev/null
+++ b/src/fftw3/reodft/rodft00e-r2hc-pad.c
@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rodft00e-r2hc-pad.c,v 1.1 2008/10/17 06:13:18 scuri Exp $ */
+
+/* Do a RODFT00 problem via an R2HC problem, padded antisymmetrically to
+   twice the size.  This is asymptotically a factor of ~2 worse than
+   rodft00e-r2hc.c (the algorithm used in e.g. FFTPACK and Numerical
+   Recipes), but we abandoned the latter after we discovered that it
+   has intrinsic accuracy problems. */
+
+#include "reodft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cld, *cldcpy;
+     int is;
+     int n;
+     int vl;
+     int ivs, ovs;
+} P;
+
+static void apply(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is;
+     int i, n = ego->n;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * (2*n), BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  buf[0] = 0.0;
+	  for (i = 1; i < n; ++i) {
+	       R a = I[(i-1) * is];
+	       buf[i] = -a;
+	       buf[2*n - i] = a;
+	  }
+	  buf[i] = 0.0; /* i == n, Nyquist */
+	  
+	  /* r2hc transform of size 2*n */
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  /* copy n-1 real numbers (imag. parts of hc array) from buf to O */
+	  {
+	       plan_rdft *cldcpy = (plan_rdft *) ego->cldcpy;
+	       cldcpy->apply((plan *) cldcpy, buf+2*n-1, O);
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     AWAKE(ego->cld, flg);
+     AWAKE(ego->cldcpy, flg);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cldcpy);
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(rodft00e-r2hc-pad-%d%v%(%p%)%(%p%))", 
+	      ego->n - 1, ego->vl, ego->cld, ego->cldcpy);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk <= 1
+		  && p->kind[0] == RODFT00
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p, const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0(ego, p));
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     P *pln;
+     const problem_rdft *p;
+     plan *cld = (plan *) 0, *cldcpy;
+     R *buf = (R *) 0;
+     int n;
+     int vl, ivs, ovs;
+     opcnt ops;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+	  goto nada;
+
+     p = (const problem_rdft *) p_;
+
+     n = p->sz->dims[0].n + 1;
+     A(n > 0);
+     buf = (R *) MALLOC(sizeof(R) * (2*n), BUFFERS);
+
+     cld = X(mkplan_d)(plnr,X(mkproblem_rdft_1_d)(X(mktensor_1d)(2*n,1,1), 
+						  X(mktensor_0d)(), 
+						  buf, buf, R2HC));
+     if (!cld)
+	  goto nada;
+
+     X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs);
+     cldcpy =
+	  X(mkplan_d)(plnr,
+		      X(mkproblem_rdft_1_d)(X(mktensor_0d)(),
+					    X(mktensor_1d)(n-1,-1,
+							   p->sz->dims[0].os), 
+					    buf+2*n-1,TAINT(p->O, ovs), R2HC));
+     if (!cldcpy)
+	  goto nada;
+
+     X(ifree)(buf);
+
+     pln = MKPLAN_RDFT(P, &padt, apply);
+
+     pln->n = n;
+     pln->is = p->sz->dims[0].is;
+     pln->cld = cld;
+     pln->cldcpy = cldcpy;
+     pln->vl = vl;
+     pln->ivs = ivs;
+     pln->ovs = ovs;
+     
+     X(ops_zero)(&ops);
+     ops.other = n-1 + 2*n; /* loads + stores (input -> buf) */
+
+     X(ops_zero)(&pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &cldcpy->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+
+ nada:
+     X(ifree0)(buf);
+     if (cld)
+	  X(plan_destroy_internal)(cld);  
+     return (plan *)0;
+}
+
+/* constructor */
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(rodft00e_r2hc_pad_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}
diff --git a/src/fftw3/reodft/rodft00e-r2hc.c b/src/fftw3/reodft/rodft00e-r2hc.c
new file mode 100644
index 0000000..46bb299
--- /dev/null
+++ b/src/fftw3/reodft/rodft00e-r2hc.c
@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) 2003 Matteo Frigo
+ * Copyright (c) 2003 Massachusetts Institute of Technology
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+/* $Id: rodft00e-r2hc.c,v 1.1 2008/10/17 06:13:18 scuri Exp $ */
+
+/* Do a RODFT00 problem via an R2HC problem, with some pre/post-processing.
+
+   This code uses the trick from FFTPACK, also documented in a similar
+   form by Numerical Recipes.  Unfortunately, this algorithm seems to
+   have intrinsic numerical problems (similar to those in
+   reodft11e-r2hc.c), possibly due to the fact that it multiplies its
+   input by a sine, causing a loss of precision near the zero.  For
+   transforms of 16k points, it has already lost three or four decimal
+   places of accuracy, which we deem unacceptable.
+
+   So, we have abandoned this algorithm in favor of the one in
+   rodft00-r2hc-pad.c, which unfortunately sacrifices 30-50% in speed.
+   The only other alternative in the literature that does not have
+   similar numerical difficulties seems to be the direct adaptation of
+   the Cooley-Tukey decomposition for antisymmetric data, but this
+   would require a whole new set of codelets and it's not clear that
+   it's worth it at this point. */
+
+#include "reodft.h"
+
+typedef struct {
+     solver super;
+} S;
+
+typedef struct {
+     plan_rdft super;
+     plan *cld;
+     twid *td;
+     int is, os;
+     int n;
+     int vl;
+     int ivs, ovs;
+} P;
+
+static void apply(const plan *ego_, R *I, R *O)
+{
+     const P *ego = (const P *) ego_;
+     int is = ego->is, os = ego->os;
+     int i, n = ego->n;
+     int iv, vl = ego->vl;
+     int ivs = ego->ivs, ovs = ego->ovs;
+     R *W = ego->td->W;
+     R *buf;
+
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     for (iv = 0; iv < vl; ++iv, I += ivs, O += ovs) {
+	  buf[0] = 0;
+	  for (i = 1; i < n - i; ++i) {
+	       E a, b, apb, amb;
+	       a = I[is * (i - 1)];
+	       b = I[is * ((n - i) - 1)];
+	       apb =  K(2.0) * W[i] * (a + b);
+	       amb = (a - b);
+	       buf[i] = apb + amb;
+	       buf[n - i] = apb - amb;
+	  }
+	  if (i == n - i) {
+	       buf[i] = K(4.0) * I[is * (i - 1)];
+	  }
+	  
+	  {
+	       plan_rdft *cld = (plan_rdft *) ego->cld;
+	       cld->apply((plan *) cld, buf, buf);
+	  }
+	  
+	  /* FIXME: use recursive/cascade summation for better stability? */
+	  O[0] = buf[0] * 0.5;
+	  for (i = 1; i + i < n - 1; ++i) {
+	       int k = i + i;
+	       O[os * (k - 1)] = -buf[n - i];
+	       O[os * k] = O[os * (k - 2)] + buf[i];
+	  }
+	  if (i + i == n - 1) {
+	       O[os * (n - 2)] = -buf[n - i];
+	  }
+     }
+
+     X(ifree)(buf);
+}
+
+static void awake(plan *ego_, int flg)
+{
+     P *ego = (P *) ego_;
+     static const tw_instr rodft00e_tw[] = {
+          { TW_SIN, 0, 1 },
+          { TW_NEXT, 1, 0 }
+     };
+
+     AWAKE(ego->cld, flg);
+
+     X(twiddle_awake)(flg, &ego->td, rodft00e_tw, 2*ego->n, 1, (ego->n+1)/2);
+}
+
+static void destroy(plan *ego_)
+{
+     P *ego = (P *) ego_;
+     X(plan_destroy_internal)(ego->cld);
+}
+
+static void print(const plan *ego_, printer *p)
+{
+     const P *ego = (const P *) ego_;
+     p->print(p, "(rodft00e-r2hc-%d%v%(%p%))", ego->n - 1, ego->vl, ego->cld);
+}
+
+static int applicable0(const solver *ego_, const problem *p_)
+{
+     UNUSED(ego_);
+     if (RDFTP(p_)) {
+          const problem_rdft *p = (const problem_rdft *) p_;
+          return (1
+		  && p->sz->rnk == 1
+		  && p->vecsz->rnk <= 1
+		  && p->kind[0] == RODFT00
+	       );
+     }
+
+     return 0;
+}
+
+static int applicable(const solver *ego, const problem *p, const planner *plnr)
+{
+     return (!NO_UGLYP(plnr) && applicable0(ego, p));
+}
+
+static plan *mkplan(const solver *ego_, const problem *p_, planner *plnr)
+{
+     P *pln;
+     const problem_rdft *p;
+     plan *cld;
+     R *buf;
+     int n;
+     opcnt ops;
+
+     static const plan_adt padt = {
+	  X(rdft_solve), awake, print, destroy
+     };
+
+     if (!applicable(ego_, p_, plnr))
+          return (plan *)0;
+
+     p = (const problem_rdft *) p_;
+
+     n = p->sz->dims[0].n + 1;
+     buf = (R *) MALLOC(sizeof(R) * n, BUFFERS);
+
+     cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n, 1, 1),
+                                                   X(mktensor_0d)(),
+                                                   buf, buf, R2HC));
+     X(ifree)(buf);
+     if (!cld)
+          return (plan *)0;
+
+     pln = MKPLAN_RDFT(P, &padt, apply);
+
+     pln->n = n;
+     pln->is = p->sz->dims[0].is;
+     pln->os = p->sz->dims[0].os;
+     pln->cld = cld;
+     pln->td = 0;
+     
+     X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs);
+     
+     X(ops_zero)(&ops);
+     ops.other = 4 + (n-1)/2 * 5 + (n-2)/2 * 5;
+     ops.add = (n-1)/2 * 4 + (n-2)/2 * 1;
+     ops.mul = 1 + (n-1)/2 * 2;
+     if (n % 2 == 0)
+	  ops.mul += 1;
+
+     X(ops_zero)(&pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops);
+     X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops);
+
+     return &(pln->super.super);
+}
+
+/* constructor */
+static solver *mksolver(void)
+{
+     static const solver_adt sadt = { mkplan };
+     S *slv = MKSOLVER(S, &sadt);
+     return &(slv->super);
+}
+
+void X(rodft00e_r2hc_register)(planner *p)
+{
+     REGISTER_SOLVER(p, mksolver());
+}