kc3-lang/libffi

diff --git a/configure.ac b/configure.ac
index d5414f9..2941cd7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -183,6 +183,23 @@ if test x$TARGET = xX86 || test x$TARGET = xX86_WIN32 || test x$TARGET = xX86_64
     fi
 fi
 
+if test x$TARGET = xS390; then
+    AC_CACHE_CHECK([compiler uses zarch features],
+	libffi_cv_as_s390_zarch, [
+	libffi_cv_as_s390_zarch=no
+	echo 'void foo(void) { bar(); bar(); }' > conftest.c
+	if $CC $CFLAGS -S conftest.c > /dev/null 2>&1; then
+	    if grep -q brasl conftest.s; then
+	        libffi_cv_as_s390_zarch=yes
+	    fi
+	fi
+	])
+    if test "x$libffi_cv_as_s390_zarch" = xyes; then
+	AC_DEFINE(HAVE_AS_S390_ZARCH, 1,
+		  [Define if the compiler uses zarch features.])
+    fi
+fi
+
 # On PaX enable kernels that have MPROTECT enable we can't use PROT_EXEC.
 AC_ARG_ENABLE(pax_emutramp,
   [  --enable-pax_emutramp       enable pax emulated trampolines, for we can't use PROT_EXEC],
diff --git a/src/s390/ffi.c b/src/s390/ffi.c
index 520ec7c..4035b6e 100644
--- a/src/s390/ffi.c
+++ b/src/s390/ffi.c
@@ -1,9 +1,9 @@
 /* -----------------------------------------------------------------------
    ffi.c - Copyright (c) 2000, 2007 Software AG
            Copyright (c) 2008 Red Hat, Inc
- 
+
    S390 Foreign Function Interface
- 
+
    Permission is hereby granted, free of charge, to any person obtaining
    a copy of this software and associated documentation files (the
    ``Software''), to deal in the Software without restriction, including
@@ -11,10 +11,10 @@
    distribute, sublicense, and/or sell copies of the Software, and to
    permit persons to whom the Software is furnished to do so, subject to
    the following conditions:
- 
+
    The above copyright notice and this permission notice shall be included
    in all copies or substantial portions of the Software.
- 
+
    THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
@@ -27,24 +27,23 @@
 /*                          Includes                                  */
 /*                          --------                                  */
 /*====================================================================*/
- 
+
 #include <ffi.h>
 #include <ffi_common.h>
- 
-#include <stdlib.h>
-#include <stdio.h>
- 
+#include <stdint.h>
+#include "internal.h"
+
 /*====================== End of Includes =============================*/
- 
+
 /*====================================================================*/
 /*                           Defines                                  */
 /*                           -------                                  */
 /*====================================================================*/
 
-/* Maximum number of GPRs available for argument passing.  */ 
+/* Maximum number of GPRs available for argument passing.  */
 #define MAX_GPRARGS 5
 
-/* Maximum number of FPRs available for argument passing.  */ 
+/* Maximum number of FPRs available for argument passing.  */
 #ifdef __s390x__
 #define MAX_FPRARGS 4
 #else
@@ -54,47 +53,30 @@
 /* Round to multiple of 16.  */
 #define ROUND_SIZE(size) (((size) + 15) & ~15)
 
-/* If these values change, sysv.S must be adapted!  */
-#define FFI390_RET_VOID		0
-#define FFI390_RET_STRUCT	1
-#define FFI390_RET_FLOAT	2
-#define FFI390_RET_DOUBLE	3
-#define FFI390_RET_INT32	4
-#define FFI390_RET_INT64	5
-
 /*===================== End of Defines ===============================*/
- 
-/*====================================================================*/
-/*                          Prototypes                                */
-/*                          ----------                                */
-/*====================================================================*/
- 
-static void ffi_prep_args (unsigned char *, extended_cif *);
-void
-#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
-__attribute__ ((visibility ("hidden")))
-#endif
-ffi_closure_helper_SYSV (ffi_closure *, unsigned long *, 
-			 unsigned long long *, unsigned long *);
 
-/*====================== End of Prototypes ===========================*/
- 
 /*====================================================================*/
 /*                          Externals                                 */
 /*                          ---------                                 */
 /*====================================================================*/
- 
-extern void ffi_call_SYSV(unsigned,
-			  extended_cif *,
-			  void (*)(unsigned char *, extended_cif *),
-			  unsigned,
-			  void *,
-			  void (*fn)(void));
+
+struct call_frame
+{
+  void *back_chain;
+  void *eos;
+  unsigned long gpr_args[5];
+  unsigned long gpr_save[9];
+  unsigned long long fpr_args[4];
+};
+
+extern void FFI_HIDDEN ffi_call_SYSV(struct call_frame *, unsigned, void *,
+			             void (*fn)(void), void *);
 
 extern void ffi_closure_SYSV(void);
- 
+extern void ffi_go_closure_SYSV(void);
+
 /*====================== End of Externals ============================*/
- 
+
 /*====================================================================*/
 /*                                                                    */
 /* Name     - ffi_check_struct_type.                                  */
@@ -103,7 +85,7 @@ extern void ffi_closure_SYSV(void);
 /*            general purpose or floating point register.             */
 /*                                                                    */
 /*====================================================================*/
- 
+
 static int
 ffi_check_struct_type (ffi_type *arg)
 {
@@ -111,7 +93,7 @@ ffi_check_struct_type (ffi_type *arg)
 
   /* If the struct has just one element, look at that element
      to find out whether to consider the struct as floating point.  */
-  while (arg->type == FFI_TYPE_STRUCT 
+  while (arg->type == FFI_TYPE_STRUCT
          && arg->elements[0] && !arg->elements[1])
     arg = arg->elements[0];
 
@@ -144,193 +126,9 @@ ffi_check_struct_type (ffi_type *arg)
   /* Other structs are passed via a pointer to the data.  */
   return FFI_TYPE_POINTER;
 }
- 
-/*======================== End of Routine ============================*/
- 
-/*====================================================================*/
-/*                                                                    */
-/* Name     - ffi_prep_args.                                          */
-/*                                                                    */
-/* Function - Prepare parameters for call to function.                */
-/*                                                                    */
-/* ffi_prep_args is called by the assembly routine once stack space   */
-/* has been allocated for the function's arguments.                   */
-/*                                                                    */
-/*====================================================================*/
- 
-static void
-ffi_prep_args (unsigned char *stack, extended_cif *ecif)
-{
-  /* The stack space will be filled with those areas:
-
-	FPR argument register save area     (highest addresses)
-	GPR argument register save area
-	temporary struct copies
-	overflow argument area              (lowest addresses)
-
-     We set up the following pointers:
-
-        p_fpr: bottom of the FPR area (growing upwards)
-	p_gpr: bottom of the GPR area (growing upwards)
-	p_ov: bottom of the overflow area (growing upwards)
-	p_struct: top of the struct copy area (growing downwards)
-
-     All areas are kept aligned to twice the word size.  */
-
-  int gpr_off = ecif->cif->bytes;
-  int fpr_off = gpr_off + ROUND_SIZE (MAX_GPRARGS * sizeof (long));
-
-  unsigned long long *p_fpr = (unsigned long long *)(stack + fpr_off);
-  unsigned long *p_gpr = (unsigned long *)(stack + gpr_off);
-  unsigned char *p_struct = (unsigned char *)p_gpr;
-  unsigned long *p_ov = (unsigned long *)stack;
-
-  int n_fpr = 0;
-  int n_gpr = 0;
-  int n_ov = 0;
-
-  ffi_type **ptr;
-  void **p_argv = ecif->avalue;
-  int i;
- 
-  /* If we returning a structure then we set the first parameter register
-     to the address of where we are returning this structure.  */
-
-  if (ecif->cif->flags == FFI390_RET_STRUCT)
-    p_gpr[n_gpr++] = (unsigned long) ecif->rvalue;
-
-  /* Now for the arguments.  */
- 
-  for (ptr = ecif->cif->arg_types, i = ecif->cif->nargs;
-       i > 0;
-       i--, ptr++, p_argv++)
-    {
-      void *arg = *p_argv;
-      int type = (*ptr)->type;
-
-#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
-      /* 16-byte long double is passed like a struct.  */
-      if (type == FFI_TYPE_LONGDOUBLE)
-	type = FFI_TYPE_STRUCT;
-#endif
-
-      /* Check how a structure type is passed.  */
-      if (type == FFI_TYPE_STRUCT || type == FFI_TYPE_COMPLEX)
-	{
-	  if (type == FFI_TYPE_COMPLEX)
-	    type = FFI_TYPE_POINTER;
-	  else
-	    type = ffi_check_struct_type (*ptr);
-
-	  /* If we pass the struct via pointer, copy the data.  */
-	  if (type == FFI_TYPE_POINTER)
-	    {
-	      p_struct -= ROUND_SIZE ((*ptr)->size);
-	      memcpy (p_struct, (char *)arg, (*ptr)->size);
-	      arg = &p_struct;
-	    }
-	}
-
-      /* Now handle all primitive int/pointer/float data types.  */
-      switch (type) 
-	{
-	  case FFI_TYPE_DOUBLE:
-	    if (n_fpr < MAX_FPRARGS)
-	      p_fpr[n_fpr++] = *(unsigned long long *) arg;
-	    else
-#ifdef __s390x__
-	      p_ov[n_ov++] = *(unsigned long *) arg;
-#else
-	      p_ov[n_ov++] = ((unsigned long *) arg)[0],
-	      p_ov[n_ov++] = ((unsigned long *) arg)[1];
-#endif
-	    break;
-	
-	  case FFI_TYPE_FLOAT:
-	    if (n_fpr < MAX_FPRARGS)
-	      p_fpr[n_fpr++] = (long long) *(unsigned int *) arg << 32;
-	    else
-	      p_ov[n_ov++] = *(unsigned int *) arg;
-	    break;
-
-	  case FFI_TYPE_POINTER:
-	    if (n_gpr < MAX_GPRARGS)
-	      p_gpr[n_gpr++] = (unsigned long)*(unsigned char **) arg;
-	    else
-	      p_ov[n_ov++] = (unsigned long)*(unsigned char **) arg;
-	    break;
- 
-	  case FFI_TYPE_UINT64:
-	  case FFI_TYPE_SINT64:
-#ifdef __s390x__
-	    if (n_gpr < MAX_GPRARGS)
-	      p_gpr[n_gpr++] = *(unsigned long *) arg;
-	    else
-	      p_ov[n_ov++] = *(unsigned long *) arg;
-#else
-	    if (n_gpr == MAX_GPRARGS-1)
-	      n_gpr = MAX_GPRARGS;
-	    if (n_gpr < MAX_GPRARGS)
-	      p_gpr[n_gpr++] = ((unsigned long *) arg)[0],
-	      p_gpr[n_gpr++] = ((unsigned long *) arg)[1];
-	    else
-	      p_ov[n_ov++] = ((unsigned long *) arg)[0],
-	      p_ov[n_ov++] = ((unsigned long *) arg)[1];
-#endif
-	    break;
- 
-	  case FFI_TYPE_UINT32:
-	    if (n_gpr < MAX_GPRARGS)
-	      p_gpr[n_gpr++] = *(unsigned int *) arg;
-	    else
-	      p_ov[n_ov++] = *(unsigned int *) arg;
-	    break;
- 
-	  case FFI_TYPE_INT:
-	  case FFI_TYPE_SINT32:
-	    if (n_gpr < MAX_GPRARGS)
-	      p_gpr[n_gpr++] = *(signed int *) arg;
-	    else
-	      p_ov[n_ov++] = *(signed int *) arg;
-	    break;
- 
-	  case FFI_TYPE_UINT16:
-	    if (n_gpr < MAX_GPRARGS)
-	      p_gpr[n_gpr++] = *(unsigned short *) arg;
-	    else
-	      p_ov[n_ov++] = *(unsigned short *) arg;
-	    break;
- 
-	  case FFI_TYPE_SINT16:
-	    if (n_gpr < MAX_GPRARGS)
-	      p_gpr[n_gpr++] = *(signed short *) arg;
-	    else
-	      p_ov[n_ov++] = *(signed short *) arg;
-	    break;
-
-	  case FFI_TYPE_UINT8:
-	    if (n_gpr < MAX_GPRARGS)
-	      p_gpr[n_gpr++] = *(unsigned char *) arg;
-	    else
-	      p_ov[n_ov++] = *(unsigned char *) arg;
-	    break;
- 
-	  case FFI_TYPE_SINT8:
-	    if (n_gpr < MAX_GPRARGS)
-	      p_gpr[n_gpr++] = *(signed char *) arg;
-	    else
-	      p_ov[n_ov++] = *(signed char *) arg;
-	    break;
- 
-	  default:
-	    FFI_ASSERT (0);
-	    break;
-        }
-    }
-}
 
 /*======================== End of Routine ============================*/
- 
+
 /*====================================================================*/
 /*                                                                    */
 /* Name     - ffi_prep_cif_machdep.                                   */
@@ -338,8 +136,8 @@ ffi_prep_args (unsigned char *stack, extended_cif *ecif)
 /* Function - Perform machine dependent CIF processing.               */
 /*                                                                    */
 /*====================================================================*/
- 
-ffi_status
+
+ffi_status FFI_HIDDEN
 ffi_prep_cif_machdep(ffi_cif *cif)
 {
   size_t struct_size = 0;
@@ -350,7 +148,7 @@ ffi_prep_cif_machdep(ffi_cif *cif)
   ffi_type **ptr;
   int i;
 
-  /* Determine return value handling.  */ 
+  /* Determine return value handling.  */
 
   switch (cif->rtype->type)
     {
@@ -364,7 +162,7 @@ ffi_prep_cif_machdep(ffi_cif *cif)
       case FFI_TYPE_COMPLEX:
 	cif->flags = FFI390_RET_STRUCT;
 	n_gpr++;  /* We need one GPR to pass the pointer.  */
-	break; 
+	break;
 
       /* Floating point values are returned in fpr 0.  */
       case FFI_TYPE_FLOAT:
@@ -403,14 +201,14 @@ ffi_prep_cif_machdep(ffi_cif *cif)
 	cif->flags = FFI390_RET_INT32;
 #endif
 	break;
- 
+
       default:
         FFI_ASSERT (0);
         break;
     }
 
   /* Now for the arguments.  */
- 
+
   for (ptr = cif->arg_types, i = cif->nargs;
        i > 0;
        i--, ptr++)
@@ -438,7 +236,7 @@ ffi_prep_cif_machdep(ffi_cif *cif)
 	}
 
       /* Now handle all primitive int/float data types.  */
-      switch (type) 
+      switch (type)
 	{
 	  /* The first MAX_FPRARGS floating point arguments
 	     go in FPRs, the rest overflow to the stack.  */
@@ -449,7 +247,7 @@ ffi_prep_cif_machdep(ffi_cif *cif)
 	    else
 	      n_ov += sizeof (double) / sizeof (long);
 	    break;
-	
+
 	  case FFI_TYPE_FLOAT:
 	    if (n_fpr < MAX_FPRARGS)
 	      n_fpr++;
@@ -459,9 +257,9 @@ ffi_prep_cif_machdep(ffi_cif *cif)
 
 	  /* On 31-bit machines, 64-bit integers are passed in GPR pairs,
 	     if one is still available, or else on the stack.  If only one
-	     register is free, skip the register (it won't be used for any 
+	     register is free, skip the register (it won't be used for any
 	     subsequent argument either).  */
-	      
+
 #ifndef __s390x__
 	  case FFI_TYPE_UINT64:
 	  case FFI_TYPE_SINT64:
@@ -477,7 +275,7 @@ ffi_prep_cif_machdep(ffi_cif *cif)
 	  /* Everything else is passed in GPRs (until MAX_GPRARGS
 	     have been used) or overflows to the stack.  */
 
-	  default: 
+	  default:
 	    if (n_gpr < MAX_GPRARGS)
 	      n_gpr++;
 	    else
@@ -490,12 +288,12 @@ ffi_prep_cif_machdep(ffi_cif *cif)
      and temporary structure copies.  */
 
   cif->bytes = ROUND_SIZE (n_ov * sizeof (long)) + struct_size;
- 
+
   return FFI_OK;
 }
- 
+
 /*======================== End of Routine ============================*/
- 
+
 /*====================================================================*/
 /*                                                                    */
 /* Name     - ffi_call.                                               */
@@ -503,42 +301,195 @@ ffi_prep_cif_machdep(ffi_cif *cif)
 /* Function - Call the FFI routine.                                   */
 /*                                                                    */
 /*====================================================================*/
- 
-void
-ffi_call(ffi_cif *cif,
-	 void (*fn)(void),
-	 void *rvalue,
-	 void **avalue)
+
+static void
+ffi_call_int(ffi_cif *cif,
+	     void (*fn)(void),
+	     void *rvalue,
+	     void **avalue,
+	     void *closure)
 {
   int ret_type = cif->flags;
-  extended_cif ecif;
- 
-  ecif.cif    = cif;
-  ecif.avalue = avalue;
-  ecif.rvalue = rvalue;
+  size_t rsize = 0, bytes = cif->bytes;
+  unsigned char *stack, *p_struct;
+  struct call_frame *frame;
+  unsigned long *p_ov, *p_gpr;
+  unsigned long long *p_fpr;
+  int n_fpr, n_gpr, n_ov, i, n;
+  ffi_type **arg_types;
+
+  FFI_ASSERT (cif->abi == FFI_SYSV);
 
   /* If we don't have a return value, we need to fake one.  */
   if (rvalue == NULL)
     {
-      if (ret_type == FFI390_RET_STRUCT)
-	ecif.rvalue = alloca (cif->rtype->size);
+      if (ret_type & FFI390_RET_IN_MEM)
+	rsize = cif->rtype->size;
       else
 	ret_type = FFI390_RET_VOID;
-    } 
+    }
+
+  /* The stack space will be filled with those areas:
 
-  switch (cif->abi)
+	dummy structure return		    (highest addresses)
+	  FPR argument register save area
+	  GPR argument register save area
+	stack frame for ffi_call_SYSV
+	temporary struct copies
+	overflow argument area              (lowest addresses)
+
+     We set up the following pointers:
+
+        p_fpr: bottom of the FPR area (growing upwards)
+	p_gpr: bottom of the GPR area (growing upwards)
+	p_ov: bottom of the overflow area (growing upwards)
+	p_struct: top of the struct copy area (growing downwards)
+
+     All areas are kept aligned to twice the word size.
+
+     Note that we're going to create the stack frame for both
+     ffi_call_SYSV _and_ the target function right here.  This
+     works because we don't make any function calls with more
+     than 5 arguments (indeed only memcpy and ffi_call_SYSV),
+     and thus we don't have any stacked outgoing parameters.  */
+
+  stack = alloca (bytes + sizeof(struct call_frame) + rsize);
+  frame = (struct call_frame *)(stack + bytes);
+  if (rsize)
+    rvalue = frame + 1;
+
+  /* Link the new frame back to the one from this function.  */
+  frame->back_chain = __builtin_frame_address (0);
+
+  /* Fill in all of the argument stuff.  */
+  p_ov = (unsigned long *)stack;
+  p_struct = (unsigned char *)frame;
+  p_gpr = frame->gpr_args;
+  p_fpr = frame->fpr_args;
+  n_fpr = n_gpr = n_ov = 0;
+
+  /* If we returning a structure then we set the first parameter register
+     to the address of where we are returning this structure.  */
+  if (cif->flags & FFI390_RET_IN_MEM)
+    p_gpr[n_gpr++] = (uintptr_t) rvalue;
+
+  /* Now for the arguments.  */
+  arg_types = cif->arg_types;
+  for (i = 0, n = cif->nargs; i < n; ++i)
     {
-      case FFI_SYSV:
-        ffi_call_SYSV (cif->bytes, &ecif, ffi_prep_args,
-		       ret_type, ecif.rvalue, fn);
-        break;
- 
-      default:
-        FFI_ASSERT (0);
-        break;
+      ffi_type *ty = arg_types[i];
+      void *arg = avalue[i];
+      int type = ty->type;
+      ffi_arg val;
+
+    restart:
+      switch (type)
+	{
+	case FFI_TYPE_SINT8:
+	  val = *(SINT8 *)arg;
+	  goto do_int;
+	case FFI_TYPE_UINT8:
+	  val = *(UINT8 *)arg;
+	  goto do_int;
+	case FFI_TYPE_SINT16:
+	  val = *(SINT16 *)arg;
+	  goto do_int;
+	case FFI_TYPE_UINT16:
+	  val = *(UINT16 *)arg;
+	  goto do_int;
+	case FFI_TYPE_INT:
+	case FFI_TYPE_SINT32:
+	  val = *(SINT32 *)arg;
+	  goto do_int;
+	case FFI_TYPE_UINT32:
+	  val = *(UINT32 *)arg;
+	  goto do_int;
+	case FFI_TYPE_POINTER:
+	  val = *(uintptr_t *)arg;
+	do_int:
+	  *(n_gpr < MAX_GPRARGS ? p_gpr + n_gpr++ : p_ov + n_ov++) = val;
+	  break;
+
+	case FFI_TYPE_UINT64:
+	case FFI_TYPE_SINT64:
+#ifdef __s390x__
+	  val = *(UINT64 *)arg;
+	  goto do_int;
+#else
+	  if (n_gpr == MAX_GPRARGS-1)
+	    n_gpr = MAX_GPRARGS;
+	  if (n_gpr < MAX_GPRARGS)
+	    p_gpr[n_gpr++] = ((UINT32 *) arg)[0],
+	    p_gpr[n_gpr++] = ((UINT32 *) arg)[1];
+	  else
+	    p_ov[n_ov++] = ((UINT32 *) arg)[0],
+	    p_ov[n_ov++] = ((UINT32 *) arg)[1];
+#endif
+	  break;
+
+	case FFI_TYPE_DOUBLE:
+	  if (n_fpr < MAX_FPRARGS)
+	    p_fpr[n_fpr++] = *(UINT64 *) arg;
+	  else
+	    {
+#ifdef __s390x__
+	      p_ov[n_ov++] = *(UINT64 *) arg;
+#else
+	      p_ov[n_ov++] = ((UINT32 *) arg)[0],
+	      p_ov[n_ov++] = ((UINT32 *) arg)[1];
+#endif
+	    }
+	  break;
+
+	case FFI_TYPE_FLOAT:
+	  val = *(UINT32 *)arg;
+	  if (n_fpr < MAX_FPRARGS)
+	    p_fpr[n_fpr++] = (UINT64)val << 32;
+	  else
+	    p_ov[n_ov++] = val;
+	  break;
+
+	case FFI_TYPE_STRUCT:
+          /* Check how a structure type is passed.  */
+	  type = ffi_check_struct_type (ty);
+	  /* Some structures are passed via a type they contain.  */
+	  if (type != FFI_TYPE_POINTER)
+	    goto restart;
+	  /* ... otherwise, passed by reference.  fallthru.  */
+
+#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
+	case FFI_TYPE_LONGDOUBLE:
+	  /* 16-byte long double is passed via reference.  */
+#endif
+	case FFI_TYPE_COMPLEX:
+	  /* Complex types are passed via reference.  */
+	  p_struct -= ROUND_SIZE (ty->size);
+	  memcpy (p_struct, arg, ty->size);
+	  val = (uintptr_t)p_struct;
+	  goto do_int;
+
+	default:
+	  FFI_ASSERT (0);
+	  break;
+        }
     }
+
+  ffi_call_SYSV (frame, ret_type & FFI360_RET_MASK, rvalue, fn, closure);
 }
- 
+
+void
+ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
+{
+  ffi_call_int(cif, fn, rvalue, avalue, NULL);
+}
+
+void
+ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue,
+	     void **avalue, void *closure)
+{
+  ffi_call_int(cif, fn, rvalue, avalue, closure);
+}
+
 /*======================== End of Routine ============================*/
 
 /*====================================================================*/
@@ -548,9 +499,11 @@ ffi_call(ffi_cif *cif,
 /* Function - Call a FFI closure target function.                     */
 /*                                                                    */
 /*====================================================================*/
- 
-void
-ffi_closure_helper_SYSV (ffi_closure *closure,
+
+void FFI_HIDDEN
+ffi_closure_helper_SYSV (ffi_cif *cif,
+			 void (*fun)(ffi_cif*,void*,void**,void*),
+			 void *user_data,
 			 unsigned long *p_gpr,
 			 unsigned long long *p_fpr,
 			 unsigned long *p_ov)
@@ -569,21 +522,16 @@ ffi_closure_helper_SYSV (ffi_closure *closure,
   int i;
 
   /* Allocate buffer for argument list pointers.  */
+  p_arg = avalue = alloca (cif->nargs * sizeof (void *));
 
-  p_arg = avalue = alloca (closure->cif->nargs * sizeof (void *));
-
-  /* If we returning a structure, pass the structure address 
-     directly to the target function.  Otherwise, have the target 
+  /* If we returning a structure, pass the structure address
+     directly to the target function.  Otherwise, have the target
      function store the return value to the GPR save area.  */
-
-  if (closure->cif->flags == FFI390_RET_STRUCT)
+  if (cif->flags & FFI390_RET_IN_MEM)
     rvalue = (void *) p_gpr[n_gpr++];
 
   /* Now for the arguments.  */
-
-  for (ptr = closure->cif->arg_types, i = closure->cif->nargs;
-       i > 0;
-       i--, p_arg++, ptr++)
+  for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, p_arg++, ptr++)
     {
       int deref_struct_pointer = 0;
       int type = (*ptr)->type;
@@ -602,7 +550,7 @@ ffi_closure_helper_SYSV (ffi_closure *closure,
 	  else
 	    type = ffi_check_struct_type (*ptr);
 
-	  /* If we pass the struct via pointer, remember to 
+	  /* If we pass the struct via pointer, remember to
 	     retrieve the pointer later.  */
 	  if (type == FFI_TYPE_POINTER)
 	    deref_struct_pointer = 1;
@@ -610,30 +558,32 @@ ffi_closure_helper_SYSV (ffi_closure *closure,
 
       /* Pointers are passed like UINTs of the same size.  */
       if (type == FFI_TYPE_POINTER)
+	{
 #ifdef __s390x__
-	type = FFI_TYPE_UINT64;
+	  type = FFI_TYPE_UINT64;
 #else
-	type = FFI_TYPE_UINT32;
+	  type = FFI_TYPE_UINT32;
 #endif
+	}
 
       /* Now handle all primitive int/float data types.  */
-      switch (type) 
+      switch (type)
 	{
 	  case FFI_TYPE_DOUBLE:
 	    if (n_fpr < MAX_FPRARGS)
 	      *p_arg = &p_fpr[n_fpr++];
 	    else
-	      *p_arg = &p_ov[n_ov], 
+	      *p_arg = &p_ov[n_ov],
 	      n_ov += sizeof (double) / sizeof (long);
 	    break;
-	
+
 	  case FFI_TYPE_FLOAT:
 	    if (n_fpr < MAX_FPRARGS)
 	      *p_arg = &p_fpr[n_fpr++];
 	    else
 	      *p_arg = (char *)&p_ov[n_ov++] + sizeof (long) - 4;
 	    break;
- 
+
 	  case FFI_TYPE_UINT64:
 	  case FFI_TYPE_SINT64:
 #ifdef __s390x__
@@ -650,7 +600,7 @@ ffi_closure_helper_SYSV (ffi_closure *closure,
 	      *p_arg = &p_ov[n_ov], n_ov += 2;
 #endif
 	    break;
- 
+
 	  case FFI_TYPE_INT:
 	  case FFI_TYPE_UINT32:
 	  case FFI_TYPE_SINT32:
@@ -659,7 +609,7 @@ ffi_closure_helper_SYSV (ffi_closure *closure,
 	    else
 	      *p_arg = (char *)&p_ov[n_ov++] + sizeof (long) - 4;
 	    break;
- 
+
 	  case FFI_TYPE_UINT16:
 	  case FFI_TYPE_SINT16:
 	    if (n_gpr < MAX_GPRARGS)
@@ -675,7 +625,7 @@ ffi_closure_helper_SYSV (ffi_closure *closure,
 	    else
 	      *p_arg = (char *)&p_ov[n_ov++] + sizeof (long) - 1;
 	    break;
- 
+
 	  default:
 	    FFI_ASSERT (0);
 	    break;
@@ -689,10 +639,10 @@ ffi_closure_helper_SYSV (ffi_closure *closure,
 
 
   /* Call the target function.  */
-  (closure->fun) (closure->cif, rvalue, avalue, closure->user_data);
+  (fun) (cif, rvalue, avalue, user_data);
 
   /* Convert the return value.  */
-  switch (closure->cif->rtype->type)
+  switch (cif->rtype->type)
     {
       /* Void is easy, and so is struct.  */
       case FFI_TYPE_VOID:
@@ -743,7 +693,7 @@ ffi_closure_helper_SYSV (ffi_closure *closure,
         break;
     }
 }
- 
+
 /*======================== End of Routine ============================*/
 
 /*====================================================================*/
@@ -753,7 +703,7 @@ ffi_closure_helper_SYSV (ffi_closure *closure,
 /* Function - Prepare a FFI closure.                                  */
 /*                                                                    */
 /*====================================================================*/
- 
+
 ffi_status
 ffi_prep_closure_loc (ffi_closure *closure,
 		      ffi_cif *cif,
@@ -761,32 +711,46 @@ ffi_prep_closure_loc (ffi_closure *closure,
 		      void *user_data,
 		      void *codeloc)
 {
+  static unsigned short const template[] = {
+    0x0d10,			/* basr %r1,0 */
+#ifndef __s390x__
+    0x9801, 0x1006,		/* lm %r0,%r1,6(%r1) */
+#else
+    0xeb01, 0x100e, 0x0004,	/* lmg %r0,%r1,14(%r1) */
+#endif
+    0x07f1			/* br %r1 */
+  };
+
+  unsigned long *tramp = (unsigned long *)&closure->tramp;
+
   if (cif->abi != FFI_SYSV)
     return FFI_BAD_ABI;
 
-#ifndef __s390x__
-  *(short *)&closure->tramp [0] = 0x0d10;   /* basr %r1,0 */
-  *(short *)&closure->tramp [2] = 0x9801;   /* lm %r0,%r1,6(%r1) */
-  *(short *)&closure->tramp [4] = 0x1006;
-  *(short *)&closure->tramp [6] = 0x07f1;   /* br %r1 */
-  *(long  *)&closure->tramp [8] = (long)codeloc;
-  *(long  *)&closure->tramp[12] = (long)&ffi_closure_SYSV;
-#else
-  *(short *)&closure->tramp [0] = 0x0d10;   /* basr %r1,0 */
-  *(short *)&closure->tramp [2] = 0xeb01;   /* lmg %r0,%r1,14(%r1) */
-  *(short *)&closure->tramp [4] = 0x100e;
-  *(short *)&closure->tramp [6] = 0x0004;
-  *(short *)&closure->tramp [8] = 0x07f1;   /* br %r1 */
-  *(long  *)&closure->tramp[16] = (long)codeloc;
-  *(long  *)&closure->tramp[24] = (long)&ffi_closure_SYSV;
-#endif 
- 
+  memcpy (tramp, template, sizeof(template));
+  tramp[2] = (unsigned long)codeloc;
+  tramp[3] = (unsigned long)&ffi_closure_SYSV;
+
   closure->cif = cif;
-  closure->user_data = user_data;
   closure->fun = fun;
- 
+  closure->user_data = user_data;
+
   return FFI_OK;
 }
 
 /*======================== End of Routine ============================*/
- 
+
+/* Build a Go language closure.  */
+
+ffi_status
+ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif *cif,
+		     void (*fun)(ffi_cif*,void*,void**,void*))
+{
+  if (cif->abi != FFI_SYSV)
+    return FFI_BAD_ABI;
+
+  closure->tramp = ffi_go_closure_SYSV;
+  closure->cif = cif;
+  closure->fun = fun;
+
+  return FFI_OK;
+}
diff --git a/src/s390/ffitarget.h b/src/s390/ffitarget.h
index 0e4868a..d8a4ee4 100644
--- a/src/s390/ffitarget.h
+++ b/src/s390/ffitarget.h
@@ -58,6 +58,7 @@ typedef enum ffi_abi {
 /* ---- Definitions for closures ----------------------------------------- */
 
 #define FFI_CLOSURES 1
+#define FFI_GO_CLOSURES 1
 #ifdef S390X
 #define FFI_TRAMPOLINE_SIZE 32
 #else
diff --git a/src/s390/internal.h b/src/s390/internal.h
new file mode 100644
index 0000000..b875578
--- /dev/null
+++ b/src/s390/internal.h
@@ -0,0 +1,11 @@
+/* If these values change, sysv.S must be adapted!  */
+#define FFI390_RET_DOUBLE	0
+#define FFI390_RET_FLOAT	1
+#define FFI390_RET_INT64	2
+#define FFI390_RET_INT32	3
+#define FFI390_RET_VOID		4
+
+#define FFI360_RET_MASK		7
+#define FFI390_RET_IN_MEM	8
+
+#define FFI390_RET_STRUCT	(FFI390_RET_VOID | FFI390_RET_IN_MEM)
diff --git a/src/s390/sysv.S b/src/s390/sysv.S
index 4731a31..c4b5006 100644
--- a/src/s390/sysv.S
+++ b/src/s390/sysv.S
@@ -1,9 +1,9 @@
 /* -----------------------------------------------------------------------
    sysv.S - Copyright (c) 2000 Software AG
             Copyright (c) 2008 Red Hat, Inc.
- 
+
    S390 Foreign Function Interface
- 
+
    Permission is hereby granted, free of charge, to any person obtaining
    a copy of this software and associated documentation files (the
    ``Software''), to deal in the Software without restriction, including
@@ -11,10 +11,10 @@
    distribute, sublicense, and/or sell copies of the Software, and to
    permit persons to whom the Software is furnished to do so, subject to
    the following conditions:
- 
+
    The above copyright notice and this permission notice shall be included
    in all copies or substantial portions of the Software.
- 
+
    THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
@@ -29,405 +29,296 @@
 #include <fficonfig.h>
 #include <ffi.h>
 
+	.text
+
 #ifndef __s390x__
- 
-.text
-
-	# r2:	cif->bytes
-	# r3:	&ecif
-	# r4:	ffi_prep_args
-	# r5:	ret_type
-	# r6:	ecif.rvalue
-	# ov:	fn 
- 
+
+	# r2:	frame
+	# r3:	ret_type
+	# r4:	ret_addr
+	# r5:	fun
+	# r6:	closure
+
 	# This assumes we are using gas.
+	.balign	8
 	.globl	ffi_call_SYSV
+	FFI_HIDDEN(ffi_call_SYSV)
 	.type	ffi_call_SYSV,%function
 ffi_call_SYSV:
-.LFB1:
-	stm	%r6,%r15,24(%r15)		# Save registers
-.LCFI0:
-	basr	%r13,0				# Set up base register
-.Lbase:
-	lr	%r11,%r15			# Set up frame pointer
-.LCFI1:
-	sr	%r15,%r2
-	ahi	%r15,-96-48			# Allocate stack
-	lr	%r8,%r6				# Save ecif.rvalue
-	sr	%r9,%r9
-	ic	%r9,.Ltable-.Lbase(%r13,%r5)	# Load epilog address
-	l	%r7,96(%r11)			# Load function address
-	st	%r11,0(%r15)			# Set up back chain
-	ahi	%r11,-48			# Register save area
-.LCFI2:
-
-	la	%r2,96(%r15)			# Save area
-						# r3 already holds &ecif
-	basr	%r14,%r4			# Call ffi_prep_args
-
-	lm	%r2,%r6,0(%r11)			# Load arguments
-	ld	%f0,32(%r11)
-	ld	%f2,40(%r11)
-	la	%r14,0(%r13,%r9)		# Set return address
-	br	%r7				# ... and call function
-
-.LretNone:					# Return void
-	l	%r4,48+56(%r11)
-	lm	%r6,%r15,48+24(%r11)
-	br	%r4
-
-.LretFloat:
-	l	%r4,48+56(%r11)
-	ste	%f0,0(%r8)			# Return float
-	lm	%r6,%r15,48+24(%r11)
-	br	%r4
- 
-.LretDouble:
-	l	%r4,48+56(%r11)
-	std	%f0,0(%r8)			# Return double
-	lm	%r6,%r15,48+24(%r11)
-	br	%r4
-
-.LretInt32:
-	l	%r4,48+56(%r11)
-	st	%r2,0(%r8)			# Return int
-	lm	%r6,%r15,48+24(%r11)
-	br	%r4
- 
-.LretInt64:
-	l	%r4,48+56(%r11)
-	stm	%r2,%r3,0(%r8)			# Return long long
-	lm	%r6,%r15,48+24(%r11)
-	br	%r4
- 
+	.cfi_startproc
+	st	%r6,44(%r2)			# Save registers
+	stm	%r12,%r14,48(%r2)
+	lr	%r13,%r2			# Install frame pointer
+	.cfi_rel_offset r6, 44
+	.cfi_rel_offset r12, 48
+	.cfi_rel_offset r13, 52
+	.cfi_rel_offset r14, 56
+	.cfi_def_cfa_register r13
+	st	%r2,0(%r15)			# Set up back chain
+	sla	%r3,3				# ret_type *= 8
+	lr	%r12,%r4			# Save ret_addr
+	lr	%r1,%r5				# Save fun
+	lr	%r0,%r6				# Install static chain
+
+	# Set return address, so that there is only one indirect jump.
+#ifdef HAVE_AS_S390_ZARCH
+	larl	%r14,.Ltable
+	ar	%r14,%r3
+#else
+	basr	%r14,0
+0:	la	%r14,.Ltable-0b(%r14,%r3)
+#endif
+
+	lm	%r2,%r6,8(%r13)			# Load arguments
+	ld	%f0,64(%r13)
+	ld	%f2,72(%r13)
+	br	%r1				# ... and call function
+
+	.balign	8
 .Ltable:
-	.byte	.LretNone-.Lbase		# FFI390_RET_VOID
-	.byte	.LretNone-.Lbase		# FFI390_RET_STRUCT
-	.byte	.LretFloat-.Lbase		# FFI390_RET_FLOAT
-	.byte	.LretDouble-.Lbase		# FFI390_RET_DOUBLE
-	.byte	.LretInt32-.Lbase		# FFI390_RET_INT32
-	.byte	.LretInt64-.Lbase		# FFI390_RET_INT64
+# FFI390_RET_DOUBLE
+	std	%f0,0(%r12)
+	j	.Ldone
+
+	.balign	8
+# FFI390_RET_FLOAT
+	ste	%f0,0(%r12)
+	j	.Ldone
+
+	.balign	8
+# FFI390_RET_INT64
+	st	%r3,4(%r12)
+	nop
+	# fallthru
+
+	.balign	8
+# FFI390_RET_INT32
+	st	%r2,0(%r12)
+	nop
+	# fallthru
+
+	.balign	8
+# FFI390_RET_VOID
+.Ldone:
+	l	%r14,56(%r13)
+	l	%r12,48(%r13)
+	l	%r6,44(%r13)
+	l	%r13,52(%r13)
+	.cfi_restore 14
+	.cfi_restore 13
+	.cfi_restore 12
+	.cfi_restore 6
+	.cfi_def_cfa r15, 96
+	br	%r14
+	.cfi_endproc
+	.size	 ffi_call_SYSV,.-ffi_call_SYSV
 
-.LFE1: 
-.ffi_call_SYSV_end:
-	.size	 ffi_call_SYSV,.ffi_call_SYSV_end-ffi_call_SYSV
 
+	.balign	8
+	.globl	ffi_go_closure_SYSV
+	FFI_HIDDEN(ffi_go_closure_SYSV)
+	.type	ffi_go_closure_SYSV,%function
+ffi_go_closure_SYSV:
+	.cfi_startproc
+	stm	%r2,%r6,8(%r15)			# Save arguments
+	lr	%r4,%r0				# Load closure -> user_data
+	l	%r2,4(%r4)			#   ->cif
+	l	%r3,8(%r4)			#   ->fun
+	j	.Ldoclosure
+	.cfi_endproc
 
+	.balign	8
 	.globl	ffi_closure_SYSV
+	FFI_HIDDEN(ffi_closure_SYSV)
 	.type	ffi_closure_SYSV,%function
 ffi_closure_SYSV:
-.LFB2:
+	.cfi_startproc
+	stm	%r2,%r6,8(%r15)			# Save arguments
+	lr	%r4,%r0				# Closure
+	l	%r2,16(%r4)			#   ->cif
+	l	%r3,20(%r4)			#   ->fun
+	l	%r4,24(%r4)			#   ->user_data
+.Ldoclosure:
 	stm	%r12,%r15,48(%r15)		# Save registers
-.LCFI10:
+	lr	%r12,%r15
+	.cfi_def_cfa_register r12
+	.cfi_rel_offset r6, 24
+	.cfi_rel_offset r12, 48
+	.cfi_rel_offset r13, 52
+	.cfi_rel_offset r14, 56
+	.cfi_rel_offset r15, 60
+#ifndef HAVE_AS_S390_ZARCH
 	basr	%r13,0				# Set up base register
 .Lcbase:
-	stm	%r2,%r6,8(%r15)			# Save arguments
-	std	%f0,64(%r15)
-	std	%f2,72(%r15)
-	lr	%r1,%r15			# Set up stack frame
-	ahi	%r15,-96
-.LCFI11:
-	l	%r12,.Lchelper-.Lcbase(%r13)	# Get helper function
-	lr	%r2,%r0				# Closure
-	la	%r3,8(%r1)			# GPRs
-	la	%r4,64(%r1)			# FPRs
-	la	%r5,96(%r1)			# Overflow
-	st	%r1,0(%r15)			# Set up back chain
-
-	bas	%r14,0(%r12,%r13)		# Call helper
-
-	l	%r4,96+56(%r15)
-	ld	%f0,96+64(%r15)			# Load return registers
-	lm	%r2,%r3,96+8(%r15)
-	lm	%r12,%r15,96+48(%r15)
-	br	%r4
+	l	%r1,.Lchelper-.Lcbase(%r13)	# Get helper function
+#endif
+	ahi	%r15,-96-8			# Set up stack frame
+	st	%r12,0(%r15)			# Set up back chain
+
+	std	%f0,64(%r12)			# Save fp arguments
+	std	%f2,72(%r12)
+
+	la	%r5,96(%r12)			# Overflow
+	st	%r5,96(%r15)
+	la	%r6,64(%r12)			# FPRs
+	la	%r5,8(%r12)			# GPRs
+#ifdef HAVE_AS_S390_ZARCH
+	brasl	%r14,ffi_closure_helper_SYSV
+#else
+	bas	%r14,0(%r1,%r13)		# Call helper
+#endif
 
+	lr	%r15,%r12
+	.cfi_def_cfa_register r15
+	lm	%r12,%r14,48(%r12)		# Restore saved registers
+	l	%r6,24(%r15)
+	ld	%f0,64(%r15)			# Load return registers
+	lm	%r2,%r3,8(%r15)
+	br	%r14
+	.cfi_endproc
+
+#ifndef HAVE_AS_S390_ZARCH
 	.align 4
 .Lchelper:
 	.long	ffi_closure_helper_SYSV-.Lcbase
+#endif
 
-.LFE2: 
-
-.ffi_closure_SYSV_end:
-	.size	 ffi_closure_SYSV,.ffi_closure_SYSV_end-ffi_closure_SYSV
-
-
-	.section	.eh_frame,EH_FRAME_FLAGS,@progbits
-.Lframe1:
-	.4byte	.LECIE1-.LSCIE1	# Length of Common Information Entry
-.LSCIE1:
-	.4byte	0x0	# CIE Identifier Tag
-	.byte	0x1	# CIE Version
-	.ascii "zR\0"	# CIE Augmentation
-	.uleb128 0x1	# CIE Code Alignment Factor
-	.sleb128 -4	# CIE Data Alignment Factor
-	.byte	0xe	# CIE RA Column
-	.uleb128 0x1	# Augmentation size
-	.byte	0x1b	# FDE Encoding (pcrel sdata4)
-	.byte	0xc	# DW_CFA_def_cfa
-	.uleb128 0xf
-	.uleb128 0x60
-	.align	4
-.LECIE1:
-.LSFDE1:
-	.4byte	.LEFDE1-.LASFDE1	# FDE Length
-.LASFDE1:
-	.4byte	.LASFDE1-.Lframe1	# FDE CIE offset
-	.4byte	.LFB1-.	# FDE initial location
-	.4byte	.LFE1-.LFB1	# FDE address range
-	.uleb128 0x0	# Augmentation size
-	.byte	0x4	# DW_CFA_advance_loc4
-	.4byte	.LCFI0-.LFB1
-	.byte	0x8f	# DW_CFA_offset, column 0xf
-	.uleb128 0x9
-	.byte	0x8e	# DW_CFA_offset, column 0xe
-	.uleb128 0xa
-	.byte	0x8d	# DW_CFA_offset, column 0xd
-	.uleb128 0xb
-	.byte	0x8c	# DW_CFA_offset, column 0xc
-	.uleb128 0xc
-	.byte	0x8b	# DW_CFA_offset, column 0xb
-	.uleb128 0xd
-	.byte	0x8a	# DW_CFA_offset, column 0xa
-	.uleb128 0xe
-	.byte	0x89	# DW_CFA_offset, column 0x9
-	.uleb128 0xf
-	.byte	0x88	# DW_CFA_offset, column 0x8
-	.uleb128 0x10
-	.byte	0x87	# DW_CFA_offset, column 0x7
-	.uleb128 0x11
-	.byte	0x86	# DW_CFA_offset, column 0x6
-	.uleb128 0x12
-	.byte	0x4	# DW_CFA_advance_loc4
-	.4byte	.LCFI1-.LCFI0
-	.byte	0xd	# DW_CFA_def_cfa_register
-	.uleb128 0xb
-	.byte	0x4	# DW_CFA_advance_loc4
-	.4byte	.LCFI2-.LCFI1
-	.byte	0xe	# DW_CFA_def_cfa_offset
-	.uleb128 0x90
-	.align	4
-.LEFDE1:
-.LSFDE2:
-	.4byte	.LEFDE2-.LASFDE2	# FDE Length
-.LASFDE2:
-	.4byte	.LASFDE2-.Lframe1	# FDE CIE offset
-	.4byte	.LFB2-.	# FDE initial location
-	.4byte	.LFE2-.LFB2	# FDE address range
-	.uleb128 0x0	# Augmentation size
-	.byte	0x4	# DW_CFA_advance_loc4
-	.4byte	.LCFI10-.LFB2
-	.byte	0x8f	# DW_CFA_offset, column 0xf
-	.uleb128 0x9
-	.byte	0x8e	# DW_CFA_offset, column 0xe
-	.uleb128 0xa
-	.byte	0x8d	# DW_CFA_offset, column 0xd
-	.uleb128 0xb
-	.byte	0x8c	# DW_CFA_offset, column 0xc
-	.uleb128 0xc
-	.byte	0x4	# DW_CFA_advance_loc4
-	.4byte	.LCFI11-.LCFI10
-	.byte	0xe	# DW_CFA_def_cfa_offset
-	.uleb128 0xc0
-	.align	4
-.LEFDE2:
+	.size	 ffi_closure_SYSV,.-ffi_closure_SYSV
 
 #else
- 
-.text
- 
-	# r2:	cif->bytes
-	# r3:	&ecif
-	# r4:	ffi_prep_args
-	# r5:	ret_type
-	# r6:	ecif.rvalue
-	# ov:	fn 
- 
+
+	# r2:	frame
+	# r3:	ret_type
+	# r4:	ret_addr
+	# r5:	fun
+	# r6:	closure
+
 	# This assumes we are using gas.
+	.balign	8
 	.globl	ffi_call_SYSV
+	FFI_HIDDEN(ffi_call_SYSV)
 	.type	ffi_call_SYSV,%function
 ffi_call_SYSV:
-.LFB1:
-	stmg	%r6,%r15,48(%r15)		# Save registers
-.LCFI0:
-	larl	%r13,.Lbase			# Set up base register
-	lgr	%r11,%r15			# Set up frame pointer
-.LCFI1:
-	sgr	%r15,%r2
-	aghi	%r15,-160-80			# Allocate stack
-	lgr	%r8,%r6				# Save ecif.rvalue
-	llgc	%r9,.Ltable-.Lbase(%r13,%r5)	# Load epilog address
-	lg	%r7,160(%r11)			# Load function address
-	stg	%r11,0(%r15)			# Set up back chain
-	aghi	%r11,-80			# Register save area
-.LCFI2:
-
-	la	%r2,160(%r15)			# Save area
-						# r3 already holds &ecif
-	basr	%r14,%r4			# Call ffi_prep_args
-
-	lmg	%r2,%r6,0(%r11)			# Load arguments
-	ld	%f0,48(%r11)
-	ld	%f2,56(%r11)
-	ld	%f4,64(%r11)
-	ld	%f6,72(%r11)
-	la	%r14,0(%r13,%r9)		# Set return address
-	br	%r7				# ... and call function
-
-.Lbase:
-.LretNone:					# Return void
-	lg	%r4,80+112(%r11)
-	lmg	%r6,%r15,80+48(%r11)
-	br	%r4
-
-.LretFloat:
-	lg	%r4,80+112(%r11)
-	ste	%f0,0(%r8)			# Return float
-	lmg	%r6,%r15,80+48(%r11)
-	br	%r4
- 
-.LretDouble:
-	lg	%r4,80+112(%r11)
-	std	%f0,0(%r8)			# Return double
-	lmg	%r6,%r15,80+48(%r11)
-	br	%r4
-
-.LretInt32:
-	lg	%r4,80+112(%r11)
-	st	%r2,0(%r8)			# Return int
-	lmg	%r6,%r15,80+48(%r11)
-	br	%r4
- 
-.LretInt64:
-	lg	%r4,80+112(%r11)
-	stg	%r2,0(%r8)			# Return long
-	lmg	%r6,%r15,80+48(%r11)
-	br	%r4
- 
+	.cfi_startproc
+	stg	%r6,88(%r2)			# Save registers
+	stmg	%r12,%r14,96(%r2)
+	lgr	%r13,%r2			# Install frame pointer
+	.cfi_rel_offset r6, 88
+	.cfi_rel_offset r12, 96
+	.cfi_rel_offset r13, 104
+	.cfi_rel_offset r14, 112
+	.cfi_def_cfa_register r13
+	stg	%r2,0(%r15)			# Set up back chain
+	larl	%r14,.Ltable			# Set up return address
+	slag	%r3,%r3,3			# ret_type *= 8
+	lgr	%r12,%r4			# Save ret_addr
+	lgr	%r1,%r5				# Save fun
+	lgr	%r0,%r6				# Install static chain
+	agr	%r14,%r3
+	lmg	%r2,%r6,16(%r13)		# Load arguments
+	ld	%f0,128(%r13)
+	ld	%f2,136(%r13)
+	ld	%f4,144(%r13)
+	ld	%f6,152(%r13)
+	br	%r1				# ... and call function
+
+	.balign	8
 .Ltable:
-	.byte	.LretNone-.Lbase		# FFI390_RET_VOID
-	.byte	.LretNone-.Lbase		# FFI390_RET_STRUCT
-	.byte	.LretFloat-.Lbase		# FFI390_RET_FLOAT
-	.byte	.LretDouble-.Lbase		# FFI390_RET_DOUBLE
-	.byte	.LretInt32-.Lbase		# FFI390_RET_INT32
-	.byte	.LretInt64-.Lbase		# FFI390_RET_INT64
+# FFI390_RET_DOUBLE
+	std	%f0,0(%r12)
+	j	.Ldone
+
+	.balign	8
+# FFI390_RET_DOUBLE
+	ste	%f0,0(%r12)
+	j	.Ldone
+
+	.balign	8
+# FFI390_RET_INT64
+	stg	%r2,0(%r12)
+
+	.balign	8
+# FFI390_RET_INT32
+	# Never used, as we always store type ffi_arg.
+	# But the stg above is 6 bytes and we cannot
+	# jump around this case, so fall through.
+	nop
+	nop
+
+	.balign	8
+# FFI390_RET_VOID
+.Ldone:
+	lg	%r14,112(%r13)
+	lg	%r12,96(%r13)
+	lg	%r6,88(%r13)
+	lg	%r13,104(%r13)
+	.cfi_restore r14
+	.cfi_restore r13
+	.cfi_restore r12
+	.cfi_restore r6
+	.cfi_def_cfa r15, 160
+	br	%r14
+	.cfi_endproc
+	.size	 ffi_call_SYSV,.-ffi_call_SYSV
 
-.LFE1: 
-.ffi_call_SYSV_end:
-	.size	 ffi_call_SYSV,.ffi_call_SYSV_end-ffi_call_SYSV
+
+	.balign	8
+	.globl	ffi_go_closure_SYSV
+	FFI_HIDDEN(ffi_go_closure_SYSV)
+	.type	ffi_go_closure_SYSV,%function
+ffi_go_closure_SYSV:
+	.cfi_startproc
+	stmg	%r2,%r6,16(%r15)		# Save arguments
+	lgr	%r4,%r0				# Load closure -> user_data
+	lg	%r2,8(%r4)			#   ->cif
+	lg	%r3,16(%r4)			#   ->fun
+	j	.Ldoclosure
+	.cfi_endproc
+	.size	 ffi_go_closure_SYSV,.-ffi_go_closure_SYSV
 
 
+	.balign	8
 	.globl	ffi_closure_SYSV
+	FFI_HIDDEN(ffi_closure_SYSV)
 	.type	ffi_closure_SYSV,%function
 ffi_closure_SYSV:
-.LFB2:
-	stmg	%r14,%r15,112(%r15)		# Save registers
-.LCFI10:
+	.cfi_startproc
 	stmg	%r2,%r6,16(%r15)		# Save arguments
-	std	%f0,128(%r15)
-	std	%f2,136(%r15)
-	std	%f4,144(%r15)
-	std	%f6,152(%r15)
-	lgr	%r1,%r15			# Set up stack frame
-	aghi	%r15,-160
-.LCFI11:
-	lgr	%r2,%r0				# Closure
-	la	%r3,16(%r1)			# GPRs
-	la	%r4,128(%r1)			# FPRs
-	la	%r5,160(%r1)			# Overflow
-	stg	%r1,0(%r15)			# Set up back chain
-
+	lgr	%r4,%r0				# Load closure
+	lg	%r2,32(%r4)			#   ->cif
+	lg	%r3,40(%r4)			#   ->fun
+	lg	%r4,48(%r4)			#   ->user_data
+.Ldoclosure:
+	stmg	%r13,%r15,104(%r15)		# Save registers
+	lgr	%r13,%r15
+	.cfi_def_cfa_register r13
+	.cfi_rel_offset r6, 48
+	.cfi_rel_offset r13, 104
+	.cfi_rel_offset r14, 112
+	.cfi_rel_offset r15, 120
+	aghi	%r15,-160-16			# Set up stack frame
+	stg	%r13,0(%r15)			# Set up back chain
+
+	std	%f0,128(%r13)			# Save fp arguments
+	std	%f2,136(%r13)
+	std	%f4,144(%r13)
+	std	%f6,152(%r13)
+	la	%r5,160(%r13)			# Overflow
+	stg	%r5,160(%r15)
+	la	%r6,128(%r13)			# FPRs
+	la	%r5,16(%r13)			# GPRs
 	brasl	%r14,ffi_closure_helper_SYSV	# Call helper
 
-	lg	%r14,160+112(%r15)
-	ld	%f0,160+128(%r15)		# Load return registers
-	lg	%r2,160+16(%r15)
-	la	%r15,160(%r15)
+	lgr	%r15,%r13
+	.cfi_def_cfa_register r15
+	lmg	%r13,%r14,104(%r13)		# Restore saved registers
+	lg	%r6,48(%r15)
+	ld	%f0,128(%r15)			# Load return registers
+	lg	%r2,16(%r15)
 	br	%r14
-.LFE2: 
-
-.ffi_closure_SYSV_end:
-	.size	 ffi_closure_SYSV,.ffi_closure_SYSV_end-ffi_closure_SYSV
-
-
-
-	.section	.eh_frame,EH_FRAME_FLAGS,@progbits
-.Lframe1:
-	.4byte	.LECIE1-.LSCIE1	# Length of Common Information Entry
-.LSCIE1:
-	.4byte	0x0	# CIE Identifier Tag
-	.byte	0x1	# CIE Version
-	.ascii "zR\0"	# CIE Augmentation
-	.uleb128 0x1	# CIE Code Alignment Factor
-	.sleb128 -8	# CIE Data Alignment Factor
-	.byte	0xe	# CIE RA Column
-	.uleb128 0x1	# Augmentation size
-	.byte	0x1b	# FDE Encoding (pcrel sdata4)
-	.byte	0xc	# DW_CFA_def_cfa
-	.uleb128 0xf
-	.uleb128 0xa0
-	.align	8
-.LECIE1:
-.LSFDE1:
-	.4byte	.LEFDE1-.LASFDE1	# FDE Length
-.LASFDE1:
-	.4byte	.LASFDE1-.Lframe1	# FDE CIE offset
-	.4byte	.LFB1-.	# FDE initial location
-	.4byte	.LFE1-.LFB1	# FDE address range
-	.uleb128 0x0	# Augmentation size
-	.byte	0x4	# DW_CFA_advance_loc4
-	.4byte	.LCFI0-.LFB1
-	.byte	0x8f	# DW_CFA_offset, column 0xf
-	.uleb128 0x5
-	.byte	0x8e	# DW_CFA_offset, column 0xe
-	.uleb128 0x6
-	.byte	0x8d	# DW_CFA_offset, column 0xd
-	.uleb128 0x7
-	.byte	0x8c	# DW_CFA_offset, column 0xc
-	.uleb128 0x8
-	.byte	0x8b	# DW_CFA_offset, column 0xb
-	.uleb128 0x9
-	.byte	0x8a	# DW_CFA_offset, column 0xa
-	.uleb128 0xa
-	.byte	0x89	# DW_CFA_offset, column 0x9
-	.uleb128 0xb
-	.byte	0x88	# DW_CFA_offset, column 0x8
-	.uleb128 0xc
-	.byte	0x87	# DW_CFA_offset, column 0x7
-	.uleb128 0xd
-	.byte	0x86	# DW_CFA_offset, column 0x6
-	.uleb128 0xe
-	.byte	0x4	# DW_CFA_advance_loc4
-	.4byte	.LCFI1-.LCFI0
-	.byte	0xd	# DW_CFA_def_cfa_register
-	.uleb128 0xb
-	.byte	0x4	# DW_CFA_advance_loc4
-	.4byte	.LCFI2-.LCFI1
-	.byte	0xe	# DW_CFA_def_cfa_offset
-	.uleb128 0xf0
-	.align	8
-.LEFDE1:
-.LSFDE2:
-	.4byte	.LEFDE2-.LASFDE2	# FDE Length
-.LASFDE2:
-	.4byte	.LASFDE2-.Lframe1	# FDE CIE offset
-	.4byte	.LFB2-.	# FDE initial location
-	.4byte	.LFE2-.LFB2	# FDE address range
-	.uleb128 0x0	# Augmentation size
-	.byte	0x4	# DW_CFA_advance_loc4
-	.4byte	.LCFI10-.LFB2
-	.byte	0x8f	# DW_CFA_offset, column 0xf
-	.uleb128 0x5
-	.byte	0x8e	# DW_CFA_offset, column 0xe
-	.uleb128 0x6
-	.byte	0x4	# DW_CFA_advance_loc4
-	.4byte	.LCFI11-.LCFI10
-	.byte	0xe	# DW_CFA_def_cfa_offset
-	.uleb128 0x140
-	.align	8
-.LEFDE2:
-
-#endif
+	.cfi_endproc
+	.size	 ffi_closure_SYSV,.-ffi_closure_SYSV
+#endif /* !s390x */
 
 #if defined __ELF__ && defined __linux__
 	.section	.note.GNU-stack,"",@progbits