thodg/cgminer

diff --git a/Makefile.am b/Makefile.am
index 9858ee8..bccba12 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -22,6 +22,7 @@ cgminer_SOURCES	= elist.h miner.h compat.h			\
 		  ocl.c ocl.h findnonce.c findnonce.h 		\
 		  sha256_generic.c sha256_4way.c sha256_via.c	\
 		  sha256_cryptopp.c sha256_sse2_amd64.c		\
+		  sha256_sse4_amd64.c \
 		  phatk110714.cl poclbm110717.cl
 
 cgminer_LDFLAGS	= $(PTHREAD_FLAGS)
diff --git a/main.c b/main.c
index 2c9bf16..f83c7c7 100644
--- a/main.c
+++ b/main.c
@@ -96,6 +96,7 @@ enum sha256_algos {
 	ALGO_CRYPTOPP,		/* Crypto++ (C) */
 	ALGO_CRYPTOPP_ASM32,	/* Crypto++ 32-bit assembly */
 	ALGO_SSE2_64,		/* SSE2 for x86_64 */
+	ALGO_SSE4_64,		/* SSE4 for x86_64 */
 };
 
 static const char *algo_names[] = {
@@ -113,6 +114,9 @@ static const char *algo_names[] = {
 #ifdef WANT_X8664_SSE2
 	[ALGO_SSE2_64]		= "sse2_64",
 #endif
+#ifdef WANT_X8664_SSE4
+	[ALGO_SSE4_64]		= "sse4_64",
+#endif
 };
 
 bool opt_debug = false;
@@ -131,7 +135,9 @@ int opt_vectors;
 int opt_worksize;
 int opt_scantime = 60;
 static const bool opt_time = true;
-#ifdef WANT_X8664_SSE2
+#ifdef WANT_X8664_SSE4
+static enum sha256_algos opt_algo = ALGO_SSE4_64;
+#elif WANT_X8664_SSE2
 static enum sha256_algos opt_algo = ALGO_SSE2_64;
 #else
 static enum sha256_algos opt_algo = ALGO_C;
@@ -322,6 +328,9 @@ static struct opt_table opt_config_table[] = {
 #ifdef WANT_X8664_SSE2
 		     "\n\tsse2_64\t\tSSE2 implementation for x86_64 machines"
 #endif
+#ifdef WANT_X8664_SSE4
+		     "\n\tsse4_64\t\tSSE4 implementation for x86_64 machines"
+#endif
 		),
 	OPT_WITH_ARG("--cpu-threads|-t",
 		     force_nthreads_int, opt_show_intval, &opt_n_threads,
@@ -1449,6 +1458,19 @@ static void *miner_thread(void *userdata)
 			break;
 #endif
 
+#ifdef WANT_X8664_SSE4
+		case ALGO_SSE4_64: {
+			unsigned int rc5 =
+			        scanhash_sse4_64(thr_id, work.midstate, work.data + 64,
+						 work.hash1, work.hash,
+						 work.target,
+					         max_nonce, &hashes_done,
+						 work.blk.nonce);
+			rc = (rc5 == -1) ? false : true;
+			}
+			break;
+#endif
+
 #ifdef WANT_SSE2_4WAY
 		case ALGO_4WAY: {
 			unsigned int rc4 =
diff --git a/miner.h b/miner.h
index ee6b55e..887c93e 100644
--- a/miner.h
+++ b/miner.h
@@ -57,6 +57,10 @@ void *alloca (size_t);
 #define WANT_X8664_SSE2 1
 #endif
 
+#if defined(__x86_64__) && defined(__SSE4_1__) && defined(HAS_YASM)
+#define WANT_X8664_SSE4 1
+#endif
+
 #if !defined(WIN32) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
 #define WANT_BUILTIN_BSWAP
 #else
@@ -212,6 +216,12 @@ extern int scanhash_sse2_64(int, const unsigned char *pmidstate, unsigned char *
 	uint32_t max_nonce, unsigned long *nHashesDone,
 	uint32_t nonce);
 
+extern int scanhash_sse4_64(int, const unsigned char *pmidstate, unsigned char *pdata,
+	unsigned char *phash1, unsigned char *phash,
+	const unsigned char *ptarget,
+	uint32_t max_nonce, unsigned long *nHashesDone,
+	uint32_t nonce);
+
 extern int
 timeval_subtract (struct timeval *result, struct timeval *x, struct timeval *y);
 
diff --git a/sha256_sse2_amd64.c b/sha256_sse2_amd64.c
index 08ae12d..3b34b30 100644
--- a/sha256_sse2_amd64.c
+++ b/sha256_sse2_amd64.c
@@ -24,7 +24,7 @@
 
 extern void CalcSha256_x64(__m128i *res, __m128i *data, uint32_t init[8]);
 
-uint32_t g_sha256_k[] = {
+static uint32_t g_sha256_k[] = {
     0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, /*  0 */
     0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
     0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, /*  8 */
@@ -44,7 +44,7 @@ uint32_t g_sha256_k[] = {
 };
 
 
-uint32_t g_sha256_hinit[8] =
+static uint32_t g_sha256_hinit[8] =
 {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19};
 
 __m128i g_4sha256_k[64];
diff --git a/sha256_sse4_amd64.c b/sha256_sse4_amd64.c
new file mode 100644
index 0000000..f150e8c
--- /dev/null
+++ b/sha256_sse4_amd64.c
@@ -0,0 +1,133 @@
+/*
+ * SHA-256 driver for ASM routine for x86_64 on Linux
+ * Copyright (c) Mark Crichton <crichton@gimp.org>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#include "config.h"
+
+#include "miner.h"
+
+#ifdef WANT_X8664_SSE4
+
+#include <string.h>
+#include <assert.h>
+
+#include <xmmintrin.h>
+#include <stdint.h>
+#include <stdio.h>
+
+extern void CalcSha256_x64_sse4(__m128i *res, __m128i *data, uint32_t init[8]);
+
+static uint32_t g_sha256_k[] = {
+    0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, /*  0 */
+    0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+    0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, /*  8 */
+    0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+    0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, /* 16 */
+    0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+    0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, /* 24 */
+    0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+    0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, /* 32 */
+    0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+    0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, /* 40 */
+    0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+    0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, /* 48 */
+    0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+    0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, /* 56 */
+    0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
+};
+
+
+static uint32_t g_sha256_hinit[8] =
+{0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19};
+
+__m128i g_4sha256_k[64];
+
+int scanhash_sse4_64(int thr_id, const unsigned char *pmidstate,
+	unsigned char *pdata,
+	unsigned char *phash1, unsigned char *phash,
+	const unsigned char *ptarget,
+	uint32_t max_nonce, unsigned long *nHashesDone,
+	uint32_t nonce)
+{
+    uint32_t *nNonce_p = (uint32_t *)(pdata + 12);
+    uint32_t m_midstate[8], m_w[16], m_w1[16];
+    __m128i m_4w[64], m_4hash[64], m_4hash1[64];
+    __m128i offset;
+    int i;
+
+    work_restart[thr_id].restart = 0;
+
+    /* For debugging */
+    union {
+        __m128i m;
+        uint32_t i[4];
+    } mi;
+
+    /* Message expansion */
+    memcpy(m_midstate, pmidstate, sizeof(m_midstate));
+    memcpy(m_w, pdata, sizeof(m_w)); /* The 2nd half of the data */
+    memcpy(m_w1, phash1, sizeof(m_w1));
+    memset(m_4hash, 0, sizeof(m_4hash));
+
+    /* Transmongrify */
+    for (i = 0; i < 16; i++)
+        m_4w[i] = _mm_set1_epi32(m_w[i]);
+
+    for (i = 0; i < 16; i++)
+        m_4hash1[i] = _mm_set1_epi32(m_w1[i]);
+
+    for (i = 0; i < 64; i++)
+	g_4sha256_k[i] = _mm_set1_epi32(g_sha256_k[i]);
+
+    offset = _mm_set_epi32(0x3, 0x2, 0x1, 0x0);
+
+    for (;;)
+    {
+	int j;
+
+	m_4w[3] = _mm_add_epi32(offset, _mm_set1_epi32(nonce));
+
+	/* Some optimization can be done here W.R.T. precalculating some hash */
+        CalcSha256_x64_sse4(m_4hash1, m_4w, m_midstate);
+	CalcSha256_x64_sse4(m_4hash, m_4hash1, g_sha256_hinit);
+
+	for (j = 0; j < 4; j++) {
+	    mi.m = m_4hash[7];
+	    if (unlikely(mi.i[j] == 0))
+		break;
+        }
+
+	/* If j = true, we found a hit...so check it */
+	/* Use the C version for a check... */
+	if (unlikely(j != 4)) {
+		for (i = 0; i < 8; i++) {
+		    mi.m = m_4hash[i];
+		    *(uint32_t *)&(phash)[i*4] = mi.i[j];
+		}
+
+		if (fulltest(phash, ptarget)) {
+		     *nHashesDone = nonce;
+		     *nNonce_p = nonce + j;
+		     return nonce + j;
+		}
+	}
+
+	nonce += 4;
+
+        if (unlikely((nonce >= max_nonce) || work_restart[thr_id].restart))
+        {
+            *nHashesDone = nonce;
+            return -1;
+	}
+   }
+}
+
+#endif /* WANT_X8664_SSE4 */
+
diff --git a/x86_64/Makefile.am b/x86_64/Makefile.am
index c74ddd2..a4c16fc 100644
--- a/x86_64/Makefile.am
+++ b/x86_64/Makefile.am
@@ -2,7 +2,7 @@ noinst_LIBRARIES	= libx8664.a
 
 SUFFIXES = .asm
 
-libx8664_a_SOURCES	= sha256_xmm_amd64.asm
+libx8664_a_SOURCES	= sha256_xmm_amd64.asm sha256_sse4_amd64.asm
 
 .asm.o:
 	$(YASM) -f elf64 $<
diff --git a/x86_64/sha256_sse4_amd64.asm b/x86_64/sha256_sse4_amd64.asm
new file mode 100644
index 0000000..f988019
--- /dev/null
+++ b/x86_64/sha256_sse4_amd64.asm
@@ -0,0 +1,258 @@
+;; SHA-256 for X86-64 for Linux, based off of:
+
+; (c) Ufasoft 2011 http://ufasoft.com mailto:support@ufasoft.com
+; Version 2011
+; This software is Public Domain
+
+; Significant re-write/optimisation and reordering by,
+; Neil Kettle <mu-b@digit-labs.org>
+; ~18% performance improvement
+
+; SHA-256 CPU SSE cruncher for Bitcoin Miner
+
+ALIGN 32
+BITS 64
+
+%define hash rdi
+%define data rsi
+%define init rdx
+
+; 0 = (1024 - 256) (mod (LAB_CALC_UNROLL*LAB_CALC_PARA*16))
+%define LAB_CALC_PARA	2
+%define LAB_CALC_UNROLL	8
+
+%define LAB_LOOP_UNROLL 8
+
+extern g_4sha256_k
+
+global CalcSha256_x64_sse4
+;	CalcSha256	hash(rdi), data(rsi), init(rdx)
+CalcSha256_x64_sse4:
+
+	push	rbx
+
+LAB_NEXT_NONCE:
+
+	mov	rcx, 64*4					; 256 - rcx is # of SHA-2 rounds
+	mov	rax, 16*4					; 64 - rax is where we expand to
+
+LAB_SHA:
+	push	rcx
+	lea	rcx, qword [data+rcx*4]				; + 1024
+	lea	r11, qword [data+rax*4]				; + 256
+
+LAB_CALC:
+%macro	lab_calc_blk 1
+
+	movntdqa	xmm0, [r11-(15-%1)*16]				; xmm0 = W[I-15]
+	movdqa	xmm2, xmm0					; xmm2 = W[I-15]	
+	movntdqa	xmm4, [r11-(15-(%1+1))*16]			; xmm4 = W[I-15+1]
+	movdqa	xmm6, xmm4					; xmm6 = W[I-15+1]	
+
+	psrld	xmm0, 3						; xmm0 = W[I-15] >> 3
+	movdqa	xmm1, xmm0					; xmm1 = W[I-15] >> 3	
+	pslld	xmm2, 14					; xmm2 = W[I-15] << 14			
+	psrld	xmm4, 3						; xmm4 = W[I-15+1] >> 3
+	movdqa	xmm5, xmm4					; xmm5 = W[I-15+1] >> 3
+	psrld	xmm5, 4						; xmm5 = W[I-15+1] >> 7	
+	pxor	xmm4, xmm5					; xmm4 = (W[I-15+1] >> 3) ^ (W[I-15+1] >> 7)	
+	pslld	xmm6, 14					; xmm6 = W[I-15+1] << 14
+	psrld	xmm1, 4						; xmm1 = W[I-15] >> 7
+	pxor	xmm0, xmm1					; xmm0 = (W[I-15] >> 3) ^ (W[I-15] >> 7)
+	pxor	xmm0, xmm2					; xmm0 = (W[I-15] >> 3) ^ (W[I-15] >> 7) ^ (W[I-15] << 14)
+	psrld	xmm1, 11					; xmm1 = W[I-15] >> 18
+	psrld	xmm5, 11					; xmm5 = W[I-15+1] >> 18
+	pxor	xmm4, xmm6					; xmm4 = (W[I-15+1] >> 3) ^ (W[I-15+1] >> 7) ^ (W[I-15+1] << 14)
+	pxor	xmm4, xmm5					; xmm4 = (W[I-15+1] >> 3) ^ (W[I-15+1] >> 7) ^ (W[I-15+1] << 14) ^ (W[I-15+1] >> 18)	
+	pslld	xmm2, 11					; xmm2 = W[I-15] << 25
+	pslld	xmm6, 11					; xmm6 = W[I-15+1] << 25
+	pxor	xmm4, xmm6					; xmm4 = (W[I-15+1] >> 3) ^ (W[I-15+1] >> 7) ^ (W[I-15+1] << 14) ^ (W[I-15+1] >> 18) ^ (W[I-15+1] << 25)
+	pxor	xmm0, xmm1					; xmm0 = (W[I-15] >> 3) ^ (W[I-15] >> 7) ^ (W[I-15] << 14) ^ (W[I-15] >> 18)
+	pxor	xmm0, xmm2					; xmm0 = (W[I-15] >> 3) ^ (W[I-15] >> 7) ^ (W[I-15] << 14) ^ (W[I-15] >> 18) ^ (W[I-15] << 25)
+	paddd	xmm0, [r11-(16-%1)*16]				; xmm0 = s0(W[I-15]) + W[I-16]
+	paddd	xmm4, [r11-(16-(%1+1))*16]			; xmm4 = s0(W[I-15+1]) + W[I-16+1]
+	movntdqa	xmm3, [r11-(2-%1)*16]				; xmm3 = W[I-2]
+	movntdqa	xmm7, [r11-(2-(%1+1))*16]			; xmm7 = W[I-2+1]
+
+;;;;;;;;;;;;;;;;;;
+
+	movdqa	xmm2, xmm3					; xmm2 = W[I-2]
+	psrld	xmm3, 10					; xmm3 = W[I-2] >> 10
+	movdqa	xmm1, xmm3					; xmm1 = W[I-2] >> 10
+	movdqa	xmm6, xmm7					; xmm6 = W[I-2+1]
+	psrld	xmm7, 10					; xmm7 = W[I-2+1] >> 10
+	movdqa	xmm5, xmm7					; xmm5 = W[I-2+1] >> 10
+
+	paddd	xmm0, [r11-(7-%1)*16]				; xmm0 = s0(W[I-15]) + W[I-16] + W[I-7]
+	paddd	xmm4, [r11-(7-(%1+1))*16]			; xmm4 = s0(W[I-15+1]) + W[I-16+1] + W[I-7+1]
+	
+	pslld	xmm2, 13					; xmm2 = W[I-2] << 13
+	pslld	xmm6, 13					; xmm6 = W[I-2+1] << 13
+	psrld	xmm1, 7						; xmm1 = W[I-2] >> 17
+	psrld	xmm5, 7						; xmm5 = W[I-2+1] >> 17
+
+
+
+	pxor	xmm3, xmm1					; xmm3 = (W[I-2] >> 10) ^ (W[I-2] >> 17)
+	psrld	xmm1, 2						; xmm1 = W[I-2] >> 19
+	pxor	xmm3, xmm2					; xmm3 = (W[I-2] >> 10) ^ (W[I-2] >> 17) ^ (W[I-2] << 13)
+	pslld	xmm2, 2						; xmm2 = W[I-2] << 15
+	pxor	xmm7, xmm5					; xmm7 = (W[I-2+1] >> 10) ^ (W[I-2+1] >> 17)
+	psrld	xmm5, 2						; xmm5 = W[I-2+1] >> 19	
+	pxor	xmm7, xmm6					; xmm7 = (W[I-2+1] >> 10) ^ (W[I-2+1] >> 17) ^ (W[I-2+1] << 13)
+	pslld	xmm6, 2						; xmm6 = W[I-2+1] << 15
+
+
+
+	pxor	xmm3, xmm1					; xmm3 = (W[I-2] >> 10) ^ (W[I-2] >> 17) ^ (W[I-2] << 13) ^ (W[I-2] >> 19)
+	pxor	xmm3, xmm2					; xmm3 = (W[I-2] >> 10) ^ (W[I-2] >> 17) ^ (W[I-2] << 13) ^ (W[I-2] >> 19) ^ (W[I-2] << 15)
+	paddd	xmm0, xmm3					; xmm0 = s0(W[I-15]) + W[I-16] + s1(W[I-2]) + W[I-7]
+	pxor	xmm7, xmm5					; xmm7 = (W[I-2+1] >> 10) ^ (W[I-2+1] >> 17) ^ (W[I-2+1] << 13) ^ (W[I-2+1] >> 19)	
+	pxor	xmm7, xmm6					; xmm7 = (W[I-2+1] >> 10) ^ (W[I-2+1] >> 17) ^ (W[I-2+1] << 13) ^ (W[I-2+1] >> 19) ^ (W[I-2+1] << 15)
+	paddd	xmm4, xmm7					; xmm4 = s0(W[I-15+1]) + W[I-16+1] + s1(W[I-2+1]) + W[I-7+1]
+
+	movdqa	[r11+(%1*16)], xmm0
+	movdqa	[r11+((%1+1)*16)], xmm4
+%endmacro
+
+%assign i 0
+%rep    LAB_CALC_UNROLL
+        lab_calc_blk i
+%assign i i+LAB_CALC_PARA
+%endrep
+
+	add	r11, LAB_CALC_UNROLL*LAB_CALC_PARA*16
+	cmp	r11, rcx
+	jb	LAB_CALC
+
+	pop	rcx
+	mov	rax, 0
+
+; Load the init values of the message into the hash.
+
+	movntdqa	xmm7, [init]
+	pshufd	xmm5, xmm7, 0x55		; xmm5 == b
+	pshufd	xmm4, xmm7, 0xAA		; xmm4 == c
+	pshufd	xmm3, xmm7, 0xFF		; xmm3 == d
+	pshufd	xmm7, xmm7, 0			; xmm7 == a
+
+	movntdqa	xmm0, [init+4*4]
+	pshufd	xmm8, xmm0, 0x55		; xmm8 == f
+	pshufd	xmm9, xmm0, 0xAA		; xmm9 == g
+	pshufd	xmm10, xmm0, 0xFF		; xmm10 == h
+	pshufd	xmm0, xmm0, 0			; xmm0 == e
+
+LAB_LOOP:
+
+;; T t1 = h + (Rotr32(e, 6) ^ Rotr32(e, 11) ^ Rotr32(e, 25)) + ((e & f) ^ AndNot(e, g)) + Expand32<T>(g_sha256_k[j]) + w[j]
+
+%macro	lab_loop_blk 0
+	movntdqa	xmm6, [data+rax*4]
+	paddd	xmm6, g_4sha256_k[rax*4]
+	add	rax, 4
+
+	paddd	xmm6, xmm10	; +h
+
+	movdqa	xmm1, xmm0
+	movdqa	xmm2, xmm9
+	pandn	xmm1, xmm2	; ~e & g
+
+	movdqa	xmm10, xmm2	; h = g
+	movdqa	xmm2, xmm8	; f
+	movdqa	xmm9, xmm2	; g = f
+
+	pand	xmm2, xmm0	; e & f
+	pxor	xmm1, xmm2	; (e & f) ^ (~e & g)
+	movdqa	xmm8, xmm0	; f = e
+
+	paddd	xmm6, xmm1	; Ch + h + w[i] + k[i]
+
+	movdqa	xmm1, xmm0
+	psrld	xmm0, 6
+	movdqa	xmm2, xmm0
+	pslld	xmm1, 7
+	psrld	xmm2, 5
+	pxor	xmm0, xmm1
+	pxor	xmm0, xmm2
+	pslld	xmm1, 14
+	psrld	xmm2, 14
+	pxor	xmm0, xmm1
+	pxor	xmm0, xmm2
+	pslld	xmm1, 5
+	pxor	xmm0, xmm1	; Rotr32(e, 6) ^ Rotr32(e, 11) ^ Rotr32(e, 25)
+	paddd	xmm6, xmm0	; xmm6 = t1
+
+	movdqa	xmm0, xmm3	; d
+	paddd	xmm0, xmm6	; e = d+t1
+
+	movdqa	xmm1, xmm5	; =b
+	movdqa	xmm3, xmm4	; d = c
+	movdqa	xmm2, xmm4	; c
+	pand	xmm2, xmm5	; b & c
+	pand	xmm4, xmm7	; a & c
+	pand	xmm1, xmm7	; a & b
+	pxor	xmm1, xmm4
+	movdqa	xmm4, xmm5	; c = b
+	movdqa	xmm5, xmm7	; b = a
+	pxor	xmm1, xmm2	; (a & c) ^ (a & d) ^ (c & d)
+	paddd	xmm6, xmm1	; t1 + ((a & c) ^ (a & d) ^ (c & d))
+
+	movdqa	xmm2, xmm7
+	psrld	xmm7, 2
+	movdqa	xmm1, xmm7
+	pslld	xmm2, 10
+	psrld	xmm1, 11
+	pxor	xmm7, xmm2
+	pxor	xmm7, xmm1
+	pslld	xmm2, 9
+	psrld	xmm1, 9
+	pxor	xmm7, xmm2
+	pxor	xmm7, xmm1
+	pslld	xmm2, 11
+	pxor	xmm7, xmm2
+	paddd	xmm7, xmm6	; a = t1 + (Rotr32(a, 2) ^ Rotr32(a, 13) ^ Rotr32(a, 22)) + ((a & c) ^ (a & d) ^ (c & d));
+%endmacro
+
+%assign i 0
+%rep    LAB_LOOP_UNROLL
+        lab_loop_blk
+%assign i i+1
+%endrep
+
+	cmp	rax, rcx
+	jb	LAB_LOOP
+
+; Finished the 64 rounds, calculate hash and save
+
+	movntdqa	xmm1, [rdx]
+	pshufd	xmm2, xmm1, 0x55
+	paddd	xmm5, xmm2
+	pshufd	xmm6, xmm1, 0xAA
+	paddd	xmm4, xmm6
+	pshufd	xmm11, xmm1, 0xFF
+	paddd	xmm3, xmm11
+	pshufd	xmm1, xmm1, 0
+	paddd	xmm7, xmm1
+
+	movntdqa	xmm1, [rdx+4*4]
+	pshufd	xmm2, xmm1, 0x55
+	paddd	xmm8, xmm2
+	pshufd	xmm6, xmm1, 0xAA
+	paddd	xmm9, xmm6
+	pshufd	xmm11, xmm1, 0xFF
+	paddd	xmm10, xmm11
+	pshufd	xmm1, xmm1, 0
+	paddd	xmm0, xmm1
+
+	movdqa	[hash+0*16], xmm7
+	movdqa	[hash+1*16], xmm5
+	movdqa	[hash+2*16], xmm4
+	movdqa	[hash+3*16], xmm3
+	movdqa	[hash+4*16], xmm0
+	movdqa	[hash+5*16], xmm8
+	movdqa	[hash+6*16], xmm9
+	movdqa	[hash+7*16], xmm10
+
+LAB_RET:
+	pop	rbx
+	ret