/* asm-i386-gcc.h -- (C) 2006 Mark Rodenkirch, Geoffrey Reynolds.

   Inline i386 assember routines for GCC.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.
*/

#ifndef _ASM_I386_GCC_H
#define _ASM_I386_GCC_H

#include "config.h"

/* Transparent unions are a GCC extension. When a function is declared to
   take transparant union arguments the argument is actually passed as the
   type of its first element. TU64_t types declared below are used to pass
   64 bit integers to low level functions that operate on their parts.
*/
typedef union
{
  uint64_t u;
  struct { uint32_t l, h; };
} __attribute__ ((transparent_union)) TU64_t;

#endif /* _ASM_I386_GCC_H */


/* This file may be included from a number of different headers. It should
   only define FEATURE if NEED_FEATURE is defined but HAVE_FEATURE is not.
*/


#if defined(NEED_mulmod64) && !defined(HAVE_mulmod64)
#define HAVE_mulmod64

/* This code is a translation of the MSC code supplied by Mark Rodenkirch.
   It should work for a,b,p up to 62 bits.
*/
#define MOD64_GREATEST_PRIME UINT64_C(4611686018427387847)
#define MOD64_SIEVE_LIMIT "2^62"

extern int mod64_dummy; /* Used as a dummy dependency only */

static inline uint64_t mulmod64(uint64_t a, uint64_t b, TU64_t p)
{
  register uint32_t r1;  /* let GCC choose two registers r1,r2 for us */
  register uint32_t r2;  /* to clobber rather than hardcode %esi,%edi */
  uint64_t tmp;
  register uint64_t ret;

  asm ("fildll %4\n\t"
       "fildll %5\n\t"
       "mov %4,%%eax\n\t"
       "mull %5\n\t"
       "fmulp %%st(0),%%st(1)\n\t"
       "mov %%eax,%2\n\t"
       "mov %%edx,%3\n\t"
       "mov 4+%4,%%eax\n\t"
       "fmul %%st(1),%%st(0)\n\t"
       "mull %5\n\t"
       "fistpll %1\n\t"
       "add %%eax,%3\n\t"
       "mov %4,%%eax\n\t"
       "mull 4+%5\n\t"
       "add %%eax,%3\n\t"
       "mov %6,%%eax\n\t"
       "mull %1\n\t"
       "sub %%eax,%2\n\t"
       "sbb %%edx,%3\n\t"
       "mov %7,%%eax\n\t"
       "mull %1\n\t"
       "sub %%eax,%3\n\t"
       "mov %6,%%eax\n\t"
       "mull 4+%1\n\t"
#if HAVE_CMOV
       "xor %%ecx,%%ecx\n\t"
       "sub %%eax,%3\n\t"
       "mov %6,%%eax\n\t"
       "mov %7,%%edx\n\t"
       "cmovge %%ecx,%%eax\n\t"
       "cmovge %%ecx,%%edx\n\t"
       "add %2,%%eax\n\t"
       "adc %3,%%edx\n\t"
#else
       "sub %%eax,%3\n\t"
       "jge 0f\n\t"
       "mov %6,%%eax\n\t"
       "add %2,%%eax\n\t"
       "mov %7,%%edx\n\t"
       "adc %3,%%edx\n\t"
       "jmp 1f\n\t"
       "0:\n\t"
       "mov %2,%%eax\n\t"
       "mov %3,%%edx\n\t"
       "1:"
#endif
       /* tmp is written to before we finish reading from a,b,p but it
          doesn't seem to be possible to use "=&m" for memory operands,
          I hope that is because it is unnecessary.
       */
       : "=&A" (ret), "=m" (tmp), "=&r" (r1), "=&r" (r2)
       : "m" (a), "m" (b), "g" (p.l), "g" (p.h),
         "m" (mod64_dummy) /* dummy input */
#if HAVE_CMOV
       /* We should let GCC choose which register to clobber instead of
          hardcoding %ecx.
       */
       : "%ecx"
#endif
       );

  return ret;
}

#if (__GNUC__ == 3 && __GNUC_MINOR__ == 4)
/* GCC 3.4 has problems when these are inline. GCC 3.3, 4.0, 4.1 are OK. */
void mod64_init(uint64_t p);
void mod64_fini(void);
#else
static inline void mod64_init(uint64_t p)
{
  /* Marking this volatile is not enough, the fake dependency on mod64_dummy
     is necessary to prevent GCC moving it relative to mulmod64().
  */
  asm volatile ("fildll %1\n\t"
                "fld1\n\t"
                "fdivp %%st(0),%%st(1)"
                : "=m" (mod64_dummy) /* dummy output */
                : "m" (p)
                );
}

static inline void mod64_fini(void)
{
  /* Marking this volatile is not enough, the fake dependency on mod64_dummy
     is necessary to prevent GCC moving it relative to mulmod64().
  */
  asm volatile ("fstp %%st(0)"
                : "=m" (mod64_dummy) /* dummy output */
                : /* no inputs */
                );
}
#endif

static inline uint64_t pre2_mulmod64(uint64_t a, TU64_t b, TU64_t p)
{
  register uint32_t r1;
  register uint32_t r2;
  register uint64_t ret;
  uint64_t tmp;

  /* For some reason splitting up the asm results in faster code for the
     CMOV branch but slower code for the non-CMOV branch.
  */
#if HAVE_CMOV
  register double x1;
  asm ("fildll %4\n\t"
       "mov %4,%%eax\n\t"
       "mull %5\n\t"
       "mov %%eax,%1\n\t"
       "mov %%edx,%2\n\t"
       "mov 4+%4,%%eax\n\t"
       "mull %5\n\t"
       "fmul %%st(1),%%st(0)\n\t"
       "add %%eax,%2\n\t"
       "mov %4,%%eax\n\t"
       "mull %6"
       : "=&A" (ret), "=&r" (r1), "=&r" (r2), "=&t" (x1)
       : "m" (a), "rm" (b.l), "rm" (b.h),
         "m" (mod64_dummy) /* dummy input */
       );

  asm ("fistpll %1\n\t"
       "add %%eax,%3\n\t"
       "mov %4,%%eax\n\t"
       "mull %1\n\t"
       "sub %%eax,%2\n\t"
       "sbb %%edx,%3\n\t"
       "mov %5,%%eax\n\t"
       "mull %1\n\t"
       "sub %%eax,%3\n\t"
       "mov %4,%%eax\n\t"
       "mull 4+%1\n\t"

       /* For some reason GCC generates slower code here if we let it choose
          a register instead of hardcoding %ecx.
       */
       "xor %%ecx,%%ecx\n\t"
       "sub %%eax,%3\n\t"
       "mov %4,%%eax\n\t"
       "mov %5,%%edx\n\t"
       "cmovge %%ecx,%%eax\n\t"
       "cmovge %%ecx,%%edx\n\t"
       "add %2,%%eax\n\t"
       "adc %3,%%edx"

       : "+&A" (ret), "=m" (tmp), "+&r" (r1), "+&r" (r2)
       : "g" (p.l), "g" (p.h), "t" (x1),
         "m" (mod64_dummy) /* dummy input */
       : "%st", "%ecx"
       );
#else
  asm ("fildll %4\n\t"
       "mov %4,%%eax\n\t"
       "mull %5\n\t"
       "mov %%eax,%2\n\t"
       "mov %%edx,%3\n\t"
       "mov 4+%4,%%eax\n\t"
       "mull %5\n\t"
       "fmul %%st(1),%%st(0)\n\t"
       "add %%eax,%3\n\t"
       "mov %4,%%eax\n\t"

       "mull %6\n\t"
       "fistpll %1\n\t"
       "add %%eax,%3\n\t"
       "mov %7,%%eax\n\t"
       "mull %1\n\t"
       "sub %%eax,%2\n\t"
       "sbb %%edx,%3\n\t"
       "mov %8,%%eax\n\t"
       "mull %1\n\t"
       "sub %%eax,%3\n\t"
       "mov %7,%%eax\n\t"
       "mull 4+%1\n\t"

       "sub %%eax,%3\n\t"
       "jge 0f\n\t"
       "mov %7,%%eax\n\t"
       "add %2,%%eax\n\t"
       "mov %8,%%edx\n\t"
       "adc %3,%%edx\n\t"
       "jmp 1f\n\t"
       "0:\n\t"
       "mov %2,%%eax\n\t"
       "mov %3,%%edx\n\t"
       "1:"
       : "=&A" (ret), "=m" (tmp), "=&r" (r1), "=&r" (r2)
       : "m" (a), "rm" (b.l), "rm" (b.h), "g" (p.l), "g" (p.h),
         "m" (mod64_dummy) /* dummy input */
       );
#endif

  return ret;
}

#if (__GNUC__ == 3 && __GNUC_MINOR__ == 4)
/* GCC 3.4 has problems when these are inline. GCC 3.3, 4.0, 4.1 are OK. */
void pre2_mulmod64_init(uint64_t b);
void pre2_mulmod64_fini(void);
#else
static inline void pre2_mulmod64_init(uint64_t b)
{
  /* Marking this volatile is not enough, the fake dependency on mod64_dummy
     is necessary to prevent GCC moving it relative to pre2_mulmod64().
  */
  asm volatile ("fildll %1\n\t"
                "fmul %%st(1),%%st(0)"
                : "=m" (mod64_dummy) /* dummy output */
                : "m" (b)
                );
}

static inline void pre2_mulmod64_fini(void)
{
  /* Marking this volatile is not enough, the fake dependency on mod64_dummy
     is necessary to prevent GCC moving it relative to pre2_mulmod64().
  */
  asm volatile ("fstp %%st(0)"
                : "=m" (mod64_dummy) /* dummy output */
                : /* no inputs */ );
}

#endif
#endif


#if defined(NEED_rem64_32) && !defined(HAVE_rem64_32)
#define HAVE_rem64_32
static inline uint32_t rem64_32(uint64_t p64, uint32_t m32)
{
  register uint32_t quot, rem;

  asm ("divl %3"
       : "=a" (quot), "=d" (rem)
       : "A" (p64), "rm" (m32) );

  return rem;
}
#endif


#if defined(NEED_memset_fast32) && !defined(HAVE_memset_fast32)
#define HAVE_memset_fast32
/* store count copies of x starting at dst.
 */
static inline
void memset_fast32(uint_fast32_t *dst, uint_fast32_t x, uint_fast32_t count)
{
  register uint32_t tmp;
  register uint32_t *ptr;

  asm ("cld\n\t"
       "rep; stosl"
       : "=c" (tmp), "=D" (ptr),
         "=m" (*(struct { uint32_t dummy[count]; } *)dst)
       : "a" (x), "0" (count), "1" (dst)
       : "cc" );
}
#endif
