thodg/cgminer

diff --git a/API-README b/API-README
index c97dee0..54d601e 100644
--- a/API-README
+++ b/API-README
@@ -330,6 +330,9 @@ The list of requests - a (*) means it requires privileged access - and replies a
                               queue, scantime, expiry
                               N is an integer in the range 0 to 9999
 
+ substats      USBSTATS       Stats of all LIBUSB mining devices except ztex
+                              e.g. Name=MMQ,ID=0,Stat=SendWork,Count=99,...|
+
 When you enable, disable or restart a GPU or PGA, you will also get Thread messages
 in the cgminer status window
 
@@ -383,7 +386,17 @@ miner.php - an example web page to access the API
 Feature Changelog for external applications using the API:
 
 
-API V1.20
+API V1.21
+
+Added API commands:
+ 'usbstats'
+
+Modified API commands:
+ each MMQ shows up as 4 devices each with it's own stats
+
+----------
+
+API V1.20 (cgminer v2.8.5)
 
 Modified API commands:
  'pools' - add 'Has Stratum', 'Stratum Active', 'Stratum URL'
diff --git a/FPGA-README b/FPGA-README
index 9274df9..a970df7 100644
--- a/FPGA-README
+++ b/FPGA-README
@@ -6,7 +6,65 @@ ModMinerQuad (MMQ)
 ------------------
 
 The mining bitstream does not survive a power cycle, so cgminer will upload
-it, if it needs to, before it starts mining
+it, if it needs to, before it starts mining (approx 7min 40sec)
+
+The red LED also flashes while it is uploading the bitstream
+
+-
+
+When mining on windows, the driver being used will determine if mining will work.
+
+If the driver doesn't allow mining, you will get a "USB init," error message
+i.e. one of:
+ open device failed, err %d, you need to install a Windows USB driver for the device
+or
+ kernel detach failed :(
+or
+ claim interface %d failed, err %d
+
+The best solution for this is to use a tool called Zadig to set the driver:
+ http://sourceforge.net/projects/libwdi/files/zadig/
+
+This allows you set the driver for the device to be WinUSB which is usually
+required to make it work if your having problems
+
+You must also make sure you are using the latest libusb-1.0.dll supplied
+with cgminer (not the libusbx version)
+
+-
+
+There is a hidden option in cgminer to dump out a lot of information
+about USB that will help the developers to assist you if you are having
+problems:
+
+ --usb-dump 0
+
+It will only help if you have a working MMQ device attached to the computer
+
+-
+
+If the MMQ doesn't respond to cgminer at all, or the red LED isn't flashing
+then you will need to reset the MMQ
+
+The red LED should always be flashing when it is mining or ready to mine
+
+To reset the MMQ, you are best to press the left "RESET" button on the
+backplane, then unplug and replug the USB cable
+
+If your MMQ doesn't have a button on the "RESET" pad, you need to join
+the two left pads of the "RESET" pad with conductive wire to reset it.
+Cutting a small (metal) paper-clip in half works well for this
+
+Then unplug the USB cable, wait for 5 seconds, then plug it back in
+
+After you press reset, the red LED near the USB port should blink continuously
+
+If it still wont work, power off, wait for 5 seconds, then power on the MMQ
+This of course means it will upload the bitstream again when you start cgminer
+
+-
+
+Device 0 is on the power end of the board
 
 -
 
@@ -64,7 +122,7 @@ a (hack) solution to this is to blacklist the MMQ USB device in
 
 Adding 2 lines like this (just above APC) should help
 # MMQ
-ATTRS{idVendor}=="ifc9", ATTRS{idProduct}=="0003", ENV{ID_MM_DEVICE_IGNORE}="1"
+ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0003", ENV{ID_MM_DEVICE_IGNORE}="1"
 
 The change will be lost and need to be re-done, next time you update the
 modem-manager software
diff --git a/Makefile.am b/Makefile.am
index 83566c6..5f84628 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -38,7 +38,7 @@ cgminer_SOURCES := cgminer.c
 
 cgminer_SOURCES	+= elist.h miner.h compat.h bench_block.h	\
 		   util.c util.h uthash.h logging.h		\
-		   sha2.c sha2.h api.c
+		   sha2.c sha2.h api.c usbutils.h
 
 cgminer_SOURCES	+= logging.c
 
@@ -91,7 +91,7 @@ cgminer_SOURCES += driver-icarus.c
 endif
 
 if HAS_MODMINER
-cgminer_SOURCES += driver-modminer.c
+cgminer_SOURCES += driver-modminer.c usbutils.c
 bitstreamsdir = $(bindir)/bitstreams
 dist_bitstreams_DATA = bitstreams/*
 endif
diff --git a/api.c b/api.c
index ff84e3c..d59b5ca 100644
--- a/api.c
+++ b/api.c
@@ -131,7 +131,7 @@ static const char SEPARATOR = '|';
 #define SEPSTR "|"
 static const char GPUSEP = ',';
 
-static const char *APIVERSION = "1.20";
+static const char *APIVERSION = "1.21";
 static const char *DEAD = "Dead";
 #if defined(HAVE_OPENCL) || defined(HAVE_AN_FPGA)
 static const char *SICK = "Sick";
@@ -227,6 +227,7 @@ static const char *OSINFO =
 #define _MINECOIN	"COIN"
 #define _DEBUGSET	"DEBUG"
 #define _SETCONFIG	"SETCONFIG"
+#define _USBSTATS	"USBSTATS"
 
 static const char ISJSON = '{';
 #define JSON0		"{"
@@ -266,6 +267,7 @@ static const char ISJSON = '{';
 #define JSON_MINECOIN	JSON1 _MINECOIN JSON2
 #define JSON_DEBUGSET	JSON1 _DEBUGSET JSON2
 #define JSON_SETCONFIG	JSON1 _SETCONFIG JSON2
+#define JSON_USBSTATS	JSON1 _USBSTATS JSON2
 #define JSON_END	JSON4 JSON5
 
 static const char *JSON_COMMAND = "command";
@@ -369,6 +371,8 @@ static const char *JSON_PARAMETER = "parameter";
 #define MSG_INVNUM 84
 #define MSG_CONPAR 85
 #define MSG_CONVAL 86
+#define MSG_USBSTA 87
+#define MSG_NOUSTA 88
 
 enum code_severity {
 	SEVERITY_ERR,
@@ -533,6 +537,8 @@ struct CODES {
  { SEVERITY_ERR,   MSG_INVNUM,	PARAM_BOTH,	"Invalid number (%d) for '%s' range is 0-9999" },
  { SEVERITY_ERR,   MSG_CONPAR,	PARAM_NONE,	"Missing config parameters 'name,N'" },
  { SEVERITY_ERR,   MSG_CONVAL,	PARAM_STR,	"Missing config value N for '%s,N'" },
+ { SEVERITY_SUCC,  MSG_USBSTA,	PARAM_NONE,	"USB Statistics" },
+ { SEVERITY_INFO,  MSG_NOUSTA,	PARAM_NONE,	"No USB Statistics" },
  { SEVERITY_FAIL, 0, 0, NULL }
 };
 
@@ -1383,27 +1389,8 @@ static void pgastatus(int pga, bool isjson)
 			frequency = cgpu->device_ztex->freqM1 * (cgpu->device_ztex->freqM + 1);
 #endif
 #ifdef USE_MODMINER
-// TODO: a modminer has up to 4 devices but only 1 set of data for all ...
-// except 4 sets of data for temp/clock
-// So this should change in the future to just find the single temp/clock
-// if the modminer code splits the device into seperate devices later
-// For now, just display the highest temp and the average clock
-		if (cgpu->api == &modminer_api) {
-			int tc = cgpu->threads;
-			int i;
-
-			temp = 0;
-			if (tc > 4)
-				tc = 4;
-			for (i = 0; i < tc; i++) {
-				struct thr_info *thr = cgpu->thr[i];
-				struct modminer_fpga_state *state = thr->cgpu_data;
-				if (state->temp > temp)
-					temp = state->temp;
-				frequency += state->clock;
-			}
-			frequency /= (tc ? tc : 1);
-		}
+		if (cgpu->api == &modminer_api)
+			frequency = cgpu->clock;
 #endif
 
 		cgpu->utility = cgpu->accepted / ( total_secs ? total_secs : 1 ) * 60;
@@ -2990,6 +2977,49 @@ static void setconfig(__maybe_unused SOCKETTYPE c, char *param, bool isjson, __m
 	strcpy(io_buffer, message(MSG_SETCONFIG, value, param, isjson));
 }
 
+static void usbstats(__maybe_unused SOCKETTYPE c, __maybe_unused char *param, bool isjson, __maybe_unused char group)
+{
+	struct api_data *root = NULL;
+
+#ifdef USE_MODMINER
+	char buf[TMPBUFSIZ];
+	int count = 0;
+
+	root = api_usb_stats(&count);
+#endif
+
+	if (!root) {
+		strcpy(io_buffer, message(MSG_NOUSTA, 0, NULL, isjson));
+		return;
+	}
+
+#ifdef USE_MODMINER
+
+	strcpy(io_buffer, message(MSG_USBSTA, 0, NULL, isjson));
+
+	if (isjson) {
+		strcat(io_buffer, COMMA);
+		strcat(io_buffer, JSON_USBSTATS);
+	}
+
+	root = print_data(root, buf, isjson);
+	strcat(io_buffer, buf);
+
+	while (42) {
+		root = api_usb_stats(&count);
+		if (!root)
+			break;
+
+		strcat(io_buffer, COMMA);
+		root = print_data(root, buf, isjson);
+		strcat(io_buffer, buf);
+	}
+
+	if (isjson)
+		strcat(io_buffer, JSON_CLOSE);
+#endif
+}
+
 static void checkcommand(__maybe_unused SOCKETTYPE c, char *param, bool isjson, char group);
 
 struct CMDS {
@@ -3045,6 +3075,7 @@ struct CMDS {
 	{ "coin",		minecoin,	false },
 	{ "debug",		debugstate,	true },
 	{ "setconfig",		setconfig,	true },
+	{ "usbstats",		usbstats,	false },
 	{ NULL,			NULL,		false }
 };
 
diff --git a/cgminer.c b/cgminer.c
index c445c8e..8a84c20 100644
--- a/cgminer.c
+++ b/cgminer.c
@@ -146,6 +146,9 @@ bool opt_disable_pool = true;
 char *opt_icarus_options = NULL;
 char *opt_icarus_timing = NULL;
 bool opt_worktime;
+#ifdef HAVE_LIBUSB
+int opt_usbdump = -1;
+#endif
 
 char *opt_kernel_path;
 char *cgminer_path;
@@ -167,6 +170,10 @@ int gpur_thr_id;
 static int api_thr_id;
 static int total_threads;
 
+#ifdef HAVE_LIBUSB
+pthread_mutex_t cgusb_lock;
+#endif
+
 static pthread_mutex_t hash_lock;
 static pthread_mutex_t qd_lock;
 static pthread_mutex_t *stgd_lock;
@@ -1099,6 +1106,11 @@ static struct opt_table opt_config_table[] = {
 	OPT_WITH_ARG("--user|-u",
 		     set_user, NULL, NULL,
 		     "Username for bitcoin JSON-RPC server"),
+#ifdef HAVE_LIBUSB
+	OPT_WITH_ARG("--usb-dump",
+		     set_int_0_to_10, opt_show_intval, &opt_usbdump,
+		     opt_hidden),
+#endif
 #ifdef HAVE_OPENCL
 	OPT_WITH_ARG("--vectors|-v",
 		     set_vector, NULL, NULL,
@@ -6583,8 +6595,15 @@ int main(int argc, char *argv[])
 	for  (i = 0; i < argc; i++)
 		initial_args[i] = strdup(argv[i]);
 	initial_args[argc] = NULL;
+
 #ifdef HAVE_LIBUSB
-        libusb_init(NULL);
+	int err = libusb_init(NULL);
+	if (err) {
+		fprintf(stderr, "libusb_init() failed err %d", err);
+		fflush(stderr);
+		quit(1, "libusb_init() failed");
+	}
+	mutex_init(&cgusb_lock);
 #endif
 
 	mutex_init(&hash_lock);
diff --git a/configure.ac b/configure.ac
index 17fda02..a0e98b5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -321,7 +321,7 @@ fi
 
 AM_CONDITIONAL([HAS_YASM], [test x$has_yasm = xtrue])
 
-if test "x$bitforce$modminer" != xnono; then
+if test "x$bitforce" != xno; then
 	AC_ARG_WITH([libudev], [AC_HELP_STRING([--without-libudev], [Autodetect FPGAs using libudev (default enabled)])],
 		[libudev=$withval],
 		[libudev=auto]
@@ -343,7 +343,7 @@ AM_CONDITIONAL([HAVE_LIBUDEV], [test x$libudev != xno])
 
 PKG_PROG_PKG_CONFIG()
 
-if test "x$ztex" != xno; then
+if test "x$ztex$modminer" != xnono; then
   case $target in
     *-*-freebsd*)
       LIBUSB_LIBS="-lusb"
@@ -502,7 +502,7 @@ else
 	echo "  Ztex.FPGAs...........: Disabled"
 fi
 
-if test "x$bitforce$modminer" != xnono; then
+if test "x$bitforce" != xno; then
 	echo "  libudev.detection....: $libudev"
 fi
 
diff --git a/driver-modminer.c b/driver-modminer.c
index 3fbedb4..690b9b1 100644
--- a/driver-modminer.c
+++ b/driver-modminer.c
@@ -17,19 +17,26 @@
 
 #include "logging.h"
 #include "miner.h"
+#include "usbutils.h"
 #include "fpgautils.h"
 #include "util.h"
 
 #define BITSTREAM_FILENAME "fpgaminer_top_fixed7_197MHz.ncd"
 #define BISTREAM_USER_ID "\2\4$B"
 
+#define BITSTREAM_MAGIC_0 0
+#define BITSTREAM_MAGIC_1 9
+
 #define MODMINER_CUTOFF_TEMP 60.0
 #define MODMINER_OVERHEAT_TEMP 50.0
+#define MODMINER_TEMP_UP_LIMIT 48.0
 #define MODMINER_OVERHEAT_CLOCK -10
 
 #define MODMINER_HW_ERROR_PERCENT 0.75
 
-#define MODMINER_MAX_CLOCK 220
+// N.B. in the latest firmware the limit is 250
+// however the voltage/temperature risks preclude that
+#define MODMINER_MAX_CLOCK 230
 #define MODMINER_DEF_CLOCK 200
 #define MODMINER_MIN_CLOCK 160
 
@@ -37,6 +44,30 @@
 #define MODMINER_CLOCK_SET 0
 #define MODMINER_CLOCK_UP 2
 
+// Commands
+#define MODMINER_PING "\x00"
+#define MODMINER_GET_VERSION "\x01"
+#define MODMINER_FPGA_COUNT "\x02"
+// Commands + require FPGAid
+#define MODMINER_GET_IDCODE '\x03'
+#define MODMINER_GET_USERCODE '\x04'
+#define MODMINER_PROGRAM '\x05'
+#define MODMINER_SET_CLOCK '\x06'
+#define MODMINER_READ_CLOCK '\x07'
+#define MODMINER_SEND_WORK '\x08'
+#define MODMINER_CHECK_WORK '\x09'
+// One byte temperature reply
+#define MODMINER_TEMP1 '\x0a'
+// Two byte temperature reply
+#define MODMINER_TEMP2 '\x0d'
+
+// +6 bytes
+#define MODMINER_SET_REG '\x0b'
+// +2 bytes
+#define MODMINER_GET_REG '\x0c'
+
+#define FPGAID_ALL 4
+
 // Maximum how many good shares in a row means clock up
 // 96 is ~34m22s at 200MH/s
 #define MODMINER_TRY_UP 96
@@ -44,379 +75,589 @@
 // This is doubled each down clock until it reaches MODMINER_TRY_UP
 // 6 is ~2m9s at 200MH/s
 #define MODMINER_EARLY_UP 6
+// Limit when reducing shares_to_good
+#define MODMINER_MIN_BACK 12
 
 struct device_api modminer_api;
 
-static inline bool _bailout(int fd, struct cgpu_info *modminer, int prio, const char *fmt, ...)
-{
-	if (fd != -1)
-		serial_close(fd);
-	if (modminer) {
-		modminer->device_fd = -1;
-		mutex_unlock(&modminer->device_mutex);
-	}
-
-	va_list ap;
-	va_start(ap, fmt);
-	vapplog(prio, fmt, ap);
-	va_end(ap);
-	return false;
-}
-
 // 45 noops sent when detecting, in case the device was left in "start job" reading
-static const char NOOP[] = "\0\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff";
+static const char NOOP[] = MODMINER_PING "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff";
 
-static bool modminer_detect_one(const char *devpath)
+static void do_ping(struct cgpu_info *modminer)
 {
-	char buf[0x100];
-	char *devname;
-	ssize_t len;
-	int fd;
-
-#ifdef WIN32
-	fd = serial_open(devpath, 0, 10, true);
-	if (fd < 0) {
-		applog(LOG_ERR, "ModMiner detect: failed to open %s", devpath);
-		return false;
-	}
+	char buf[0x100+1];
+	int err, amount;
 
-	(void)(write(fd, NOOP, sizeof(NOOP)-1) ?:0);
-	while (serial_read(fd, buf, sizeof(buf)) > 0)
-		;
+	// Don't care if it fails
+	err = usb_write(modminer, (char *)NOOP, sizeof(NOOP)-1, &amount, C_PING);
+	applog(LOG_DEBUG, "%s%u: flush noop got %d err %d",
+		modminer->api->name, modminer->fpgaid, amount, err);
 
-	// Version
-	if (1 != write(fd, "\x01", 1)) {
-		applog(LOG_ERR, "ModMiner detect: version request failed on %s (%d)", devpath, errno);
-		goto shin;
-	}
+	// Clear any outstanding data
+	while ((err = usb_read(modminer, buf, sizeof(buf)-1, &amount, C_CLEAR)) == 0 && amount > 0)
+		applog(LOG_DEBUG, "%s%u: clear got %d",
+			modminer->api->name, modminer->fpgaid, amount);
 
-	len = serial_read(fd, buf, sizeof(buf)-1);
-	if (len < 1) {
-		applog(LOG_ERR, "ModMiner detect: no version reply on %s (%d)", devpath, errno);
-		goto shin;
-	}
-	buf[len] = '\0';
-	devname = strdup(buf);
-	applog(LOG_DEBUG, "ModMiner identified as: %s", devname);
+	applog(LOG_DEBUG, "%s%u: final clear got %d err %d",
+		modminer->api->name, modminer->fpgaid, amount, err);
+}
 
-	// FPGA count
-	if (1 != write(fd, "\x02", 1)) {
-		applog(LOG_ERR, "ModMiner detect: FPGA count request failed on %s (%d)", devpath, errno);
-		goto shin;
-	}
-	len = read(fd, buf, 1);
+static bool modminer_detect_one(struct libusb_device *dev, struct usb_find_devices *found)
+{
+	char buf[0x100+1];
+	char *devname = NULL;
+	char devpath[20];
+	int err, i, amount;
+	bool added = false;
 
-	if (len < 1) {
-		applog(LOG_ERR, "ModMiner detect: timeout waiting for FPGA count from %s (%d)", devpath, errno);
+	struct cgpu_info *modminer = NULL;
+	modminer = calloc(1, sizeof(*modminer));
+	modminer->api = &modminer_api;
+	modminer->modminer_mutex = calloc(1, sizeof(*(modminer->modminer_mutex)));
+	mutex_init(modminer->modminer_mutex);
+	modminer->fpgaid = (char)0;
+
+	if (!usb_init(modminer, dev, found))
 		goto shin;
-	}
 
-	serial_close(fd);
-#else
-	fd = select_open(devpath);
+	do_ping(modminer);
 
-	if (fd < 0) {
-		applog(LOG_ERR, "ModMiner detect: failed to open %s", devpath);
-		return false;
+	if ((err = usb_write(modminer, MODMINER_GET_VERSION, 1, &amount, C_REQUESTVERSION)) < 0 || amount != 1) {
+		applog(LOG_ERR, "ModMiner detect: send version request failed (%d:%d)", amount, err);
+		goto unshin;
 	}
 
-	// Don't care if they fail
-	select_write(fd, (char *)NOOP, sizeof(NOOP)-1);
-
-	// Will clear up to a max of sizeof(buf)-1 chars
-	select_read(fd, buf, sizeof(buf)-1);
+	if ((err = usb_read(modminer, buf, sizeof(buf)-1, &amount, C_GETVERSION)) < 0 || amount < 1) {
+		if (err < 0)
+			applog(LOG_ERR, "ModMiner detect: no version reply (%d)", err);
+		else
+			applog(LOG_ERR, "ModMiner detect: empty version reply (%d)", amount);
 
-	// Version
-	if (select_write(fd, "\x01", 1) < 1) {
-		applog(LOG_ERR, "ModMiner detect: version request failed on %s (%d)", devpath, errno);
-		goto shin;
-	}
+		applog(LOG_DEBUG, "ModMiner detect: check the firmware");
 
-	if ((len = select_read(fd, buf, sizeof(buf)-1)) < 1) {
-		applog(LOG_ERR, "ModMiner detect: no version reply on %s (%d)", devpath, errno);
-		goto shin;
+		goto unshin;
 	}
-	buf[len] = '\0';
+	buf[amount] = '\0';
 	devname = strdup(buf);
 	applog(LOG_DEBUG, "ModMiner identified as: %s", devname);
 
-	// FPGA count
-	if (select_write(fd, "\x02", 1) < 1) {
-		applog(LOG_ERR, "ModMiner detect: FPGA count request failed on %s (%d)", devpath, errno);
-		goto shin;
+	if ((err = usb_write(modminer, MODMINER_FPGA_COUNT, 1, &amount, C_REQUESTFPGACOUNT) < 0 || amount != 1)) {
+		applog(LOG_ERR, "ModMiner detect: FPGA count request failed (%d:%d)", amount, err);
+		goto unshin;
 	}
 
-	if ((len = select_read(fd, buf, 1)) < 1) {
-		applog(LOG_ERR, "ModMiner detect: no FPGA count reply on %s (%d)", devpath, errno);
-		goto shin;
+	if ((err = usb_read(modminer, buf, 1, &amount, C_GETFPGACOUNT)) < 0 || amount != 1) {
+		applog(LOG_ERR, "ModMiner detect: no FPGA count reply (%d:%d)", amount, err);
+		goto unshin;
 	}
 
-	select_close(fd);
-#endif
-
-	// TODO: check if it supports 2 byte temperatures and if not
-	// add a flag and set it use 1 byte and code to use the flag
+	// TODO: flag it use 1 byte temp if it is an old firmware
+	// can detect with modminer->cgusb->serial ?
 
 	if (buf[0] == 0) {
-		applog(LOG_ERR, "ModMiner detect: zero FPGA count from %s", devpath);
-		goto shin;
+		applog(LOG_ERR, "ModMiner detect: zero FPGA count from %s", devname);
+		goto unshin;
 	}
 
 	if (buf[0] < 1 || buf[0] > 4) {
-		applog(LOG_ERR, "ModMiner detect: invalid FPGA count (%u) from %s", buf[0], devpath);
-		goto shin;
+		applog(LOG_ERR, "ModMiner detect: invalid FPGA count (%u) from %s", buf[0], devname);
+		goto unshin;
 	}
 
 	applog(LOG_DEBUG, "ModMiner %s has %u FPGAs", devname, buf[0]);
 
-	struct cgpu_info *modminer;
-	modminer = calloc(1, sizeof(*modminer));
-	modminer->api = &modminer_api;
-	mutex_init(&modminer->device_mutex);
-	modminer->device_path = strdup(devpath);
-	modminer->device_fd = -1;
-	modminer->deven = DEV_ENABLED;
-	modminer->threads = buf[0];
 	modminer->name = devname;
 
-	return add_cgpu(modminer);
+	// TODO: test with 1 board missing in the middle and each end
+	// to see how that affects the sequence numbers
+	for (i = 0; i < buf[0]; i++) {
+		struct cgpu_info *tmp = calloc(1, sizeof(*tmp));
+
+		tmp->api = modminer->api;
+		tmp->name = devname;
+
+		sprintf(devpath, "%d:%d:%d",
+			(int)(modminer->usbdev->bus_number),
+			(int)(modminer->usbdev->device_address),
+			i);
+
+		tmp->device_path = strdup(devpath);
+		tmp->usbdev = modminer->usbdev;
+		// Only the first copy gets the already used stats
+		if (!added)
+			tmp->usbstat = modminer->usbstat;
+		tmp->fpgaid = (char)i;
+		tmp->modminer_mutex = modminer->modminer_mutex;
+		tmp->deven = DEV_ENABLED;
+		tmp->threads = 1;
+
+		if (!add_cgpu(tmp)) {
+			free(tmp->device_path);
+			free(tmp);
+			goto unshin;
+		}
+
+		update_usb_stats(tmp);
+
+		added = true;
+	}
+
+	free(modminer);
+
+	return true;
+
+unshin:
+	if (!added)
+		usb_uninit(modminer);
 
 shin:
+	if (!added)
+		free(modminer->modminer_mutex);
 
-#ifdef WIN32
-	serial_close(fd);
-#else
-	select_close(fd);
-#endif
-	return false;
+	free(modminer);
+
+	if (added)
+		return true;
+	else
+		return false;
 }
 
-static int modminer_detect_auto()
+static void modminer_detect()
 {
-	return
-	serial_autodetect_udev     (modminer_detect_one, "*ModMiner*") ?:
-	serial_autodetect_devserial(modminer_detect_one, "BTCFPGA_ModMiner") ?:
-	0;
+	usb_detect(&modminer_api, modminer_detect_one);
 }
 
-static void modminer_detect()
+static bool get_expect(struct cgpu_info *modminer, FILE *f, char c)
+{
+	char buf;
+
+	if (fread(&buf, 1, 1, f) != 1) {
+		applog(LOG_ERR, "%s%u: Error (%d) reading bitstream (%c)",
+				modminer->api->name, modminer->device_id, errno, c);
+		return false;
+	}
+
+	if (buf != c) {
+		applog(LOG_ERR, "%s%u: firmware code mismatch (%c)",
+				modminer->api->name, modminer->device_id, c);
+		return false;
+	}
+
+	return true;
+}
+
+static bool get_info(struct cgpu_info *modminer, FILE *f, char *buf, int bufsiz, const char *name)
+{
+	unsigned char siz[2];
+	int len;
+
+	if (fread(siz, 2, 1, f) != 1) {
+		applog(LOG_ERR, "%s%u: Error (%d) reading bitstream '%s' len",
+			modminer->api->name, modminer->device_id, errno, name);
+		return false;
+	}
+
+	len = siz[0] * 256 + siz[1];
+
+	if (len >= bufsiz) {
+		applog(LOG_ERR, "%s%u: Bitstream '%s' len too large (%d)",
+			modminer->api->name, modminer->device_id, name, len);
+		return false;
+	}
+
+	if (fread(buf, len, 1, f) != 1) {
+		applog(LOG_ERR, "%s%u: Error (%d) reading bitstream '%s'", errno,
+			modminer->api->name, modminer->device_id, errno, name);
+		return false;
+	}
+
+	buf[len] = '\0';
+
+	return true;
+}
+
+#define USE_DEFAULT_TIMEOUT 0
+
+// mutex must always be locked before calling
+static bool get_status_timeout(struct cgpu_info *modminer, char *msg, unsigned int timeout, enum usb_cmds cmd)
 {
-	serial_detect_auto(&modminer_api, modminer_detect_one, modminer_detect_auto);
+	int err, amount;
+	char buf[1];
+
+	if (timeout == USE_DEFAULT_TIMEOUT)
+		err = usb_read(modminer, buf, 1, &amount, cmd);
+	else
+		err = usb_read_timeout(modminer, buf, 1, &amount, timeout, cmd);
+
+	if (err < 0 || amount != 1) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Error (%d:%d) getting %s reply",
+			modminer->api->name, modminer->device_id, amount, err, msg);
+
+		return false;
+	}
+
+	if (buf[0] != 1) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Error, invalid %s reply (was %d should be 1)",
+			modminer->api->name, modminer->device_id, msg, buf[0]);
+
+		return false;
+	}
+
+	return true;
 }
 
-#define bailout(...)  return _bailout(-1, modminer, __VA_ARGS__);
-#define bailout2(...)  return _bailout(fd, modminer, __VA_ARGS__);
-
-#define check_magic(L)  do {  \
-	if (1 != fread(buf, 1, 1, f))  \
-		bailout(LOG_ERR, "Error reading ModMiner firmware ('%c')", L);  \
-	if (buf[0] != L)  \
-		bailout(LOG_ERR, "ModMiner firmware has wrong magic ('%c')", L);  \
-} while(0)
-
-#define read_str(eng)  do {  \
-	if (1 != fread(buf, 2, 1, f))  \
-		bailout(LOG_ERR, "Error reading ModMiner firmware (" eng " len)");  \
-	len = (ubuf[0] << 8) | ubuf[1];  \
-	if (len >= sizeof(buf))  \
-		bailout(LOG_ERR, "ModMiner firmware " eng " too long");  \
-	if (1 != fread(buf, len, 1, f))  \
-		bailout(LOG_ERR, "Error reading ModMiner firmware (" eng ")");  \
-	buf[len] = '\0';  \
-} while(0)
-
-#define status_read(eng)  do {  \
-FD_ZERO(&fds); \
-FD_SET(fd, &fds);  \
-select(fd+1, &fds, NULL, NULL, NULL);  \
-	if (1 != read(fd, buf, 1))  \
-		bailout2(LOG_ERR, "%s %u: Error programming %s (" eng ")", modminer->api->name, modminer->device_id, modminer->device_path);  \
-	if (buf[0] != 1)  \
-		bailout2(LOG_ERR, "%s %u: Wrong " eng " programming %s", modminer->api->name, modminer->device_id, modminer->device_path);  \
-} while(0)
+// mutex must always be locked before calling
+static bool get_status(struct cgpu_info *modminer, char *msg, enum usb_cmds cmd)
+{
+	return get_status_timeout(modminer, msg, USE_DEFAULT_TIMEOUT, cmd);
+}
 
 static bool modminer_fpga_upload_bitstream(struct cgpu_info *modminer)
 {
-	fd_set fds;
-	char buf[0x100];
+	const char *bsfile = BITSTREAM_FILENAME;
+	char buf[0x100], *p;
+	char devmsg[64];
 	unsigned char *ubuf = (unsigned char *)buf;
-	unsigned long len;
-	char *p;
-	const char *fwfile = BITSTREAM_FILENAME;
-	char fpgaid = 4;  // "all FPGAs"
-
-	FILE *f = open_bitstream("modminer", fwfile);
-	if (!f)
-		bailout(LOG_ERR, "Error opening ModMiner firmware file %s", fwfile);
-	if (1 != fread(buf, 2, 1, f))
-		bailout(LOG_ERR, "Error reading ModMiner firmware (magic)");
-	if (buf[0] || buf[1] != 9)
-		bailout(LOG_ERR, "ModMiner firmware has wrong magic (9)");
-	if (-1 == fseek(f, 11, SEEK_CUR))
-		bailout(LOG_ERR, "ModMiner firmware seek failed");
-	check_magic('a');
-	read_str("design name");
-	applog(LOG_DEBUG, "ModMiner firmware file %s info:", fwfile);
-	applog(LOG_DEBUG, "  Design name: %s", buf);
-	p = strrchr(buf, ';') ?: buf;
-	p = strrchr(buf, '=') ?: p;
+	unsigned long totlen, len;
+	size_t buflen, remaining;
+	float nextmsg, upto;
+	char fpgaid = FPGAID_ALL;
+	int err, amount, tries;
+	char *ptr;
+
+	FILE *f = open_bitstream("modminer", bsfile);
+	if (!f) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_DEBUG, "%s%u: Error (%d) opening bitstream file %s",
+			modminer->api->name, modminer->device_id, errno, bsfile);
+
+		return false;
+	}
+
+	if (fread(buf, 2, 1, f) != 1) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Error (%d) reading bitstream magic",
+			modminer->api->name, modminer->device_id, errno);
+
+		goto dame;
+	}
+
+	if (buf[0] != BITSTREAM_MAGIC_0 || buf[1] != BITSTREAM_MAGIC_1) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: bitstream has incorrect magic (%u,%u) instead of (%u,%u)",
+			modminer->api->name, modminer->device_id,
+			buf[0], buf[1],
+			BITSTREAM_MAGIC_0, BITSTREAM_MAGIC_1);
+
+		goto dame;
+	}
+
+	if (fseek(f, 11L, SEEK_CUR)) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Error (%d) bitstream seek failed",
+			modminer->api->name, modminer->device_id, errno);
+
+		goto dame;
+	}
+
+	if (!get_expect(modminer, f, 'a'))
+		goto undame;
+
+	if (!get_info(modminer, f, buf, sizeof(buf), "Design name"))
+		goto undame;
+
+	applog(LOG_DEBUG, "%s%u: bitstream file '%s' info:",
+		modminer->api->name, modminer->device_id, bsfile);
+
+	applog(LOG_DEBUG, " Design name: '%s'", buf);
+
+	p = strrchr(buf, ';') ? : buf;
+	p = strrchr(buf, '=') ? : p;
 	if (p[0] == '=')
-		++p;
+		p++;
+
 	unsigned long fwusercode = (unsigned long)strtoll(p, &p, 16);
-	if (p[0] != '\0')
-		bailout(LOG_ERR, "Bad usercode in ModMiner firmware file");
-	if (fwusercode == 0xffffffff)
-		bailout(LOG_ERR, "ModMiner firmware doesn't support user code");
-	applog(LOG_DEBUG, "  Version: %u, build %u", (fwusercode >> 8) & 0xff, fwusercode & 0xff);
-	check_magic('b');
-	read_str("part number");
-	applog(LOG_DEBUG, "  Part number: %s", buf);
-	check_magic('c');
-	read_str("build date");
-	applog(LOG_DEBUG, "  Build date: %s", buf);
-	check_magic('d');
-	read_str("build time");
-	applog(LOG_DEBUG, "  Build time: %s", buf);
-	check_magic('e');
-	if (1 != fread(buf, 4, 1, f))
-		bailout(LOG_ERR, "Error reading ModMiner firmware (data len)");
+
+	if (p[0] != '\0') {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Bad usercode in bitstream file",
+			modminer->api->name, modminer->device_id);
+
+		goto dame;
+	}
+
+	if (fwusercode == 0xffffffff) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: bitstream doesn't support user code",
+			modminer->api->name, modminer->device_id);
+
+		goto dame;
+	}
+
+	applog(LOG_DEBUG, " Version: %u, build %u", (fwusercode >> 8) & 0xff, fwusercode & 0xff);
+
+	if (!get_expect(modminer, f, 'b'))
+		goto undame;
+
+	if (!get_info(modminer, f, buf, sizeof(buf), "Part number"))
+		goto undame;
+
+	applog(LOG_DEBUG, " Part number: '%s'", buf);
+
+	if (!get_expect(modminer, f, 'c'))
+		goto undame;
+
+	if (!get_info(modminer, f, buf, sizeof(buf), "Build date"))
+		goto undame;
+
+	applog(LOG_DEBUG, " Build date: '%s'", buf);
+
+	if (!get_expect(modminer, f, 'd'))
+		goto undame;
+
+	if (!get_info(modminer, f, buf, sizeof(buf), "Build time"))
+		goto undame;
+
+	applog(LOG_DEBUG, " Build time: '%s'", buf);
+
+	if (!get_expect(modminer, f, 'e'))
+		goto undame;
+
+	if (fread(buf, 4, 1, f) != 1) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Error (%d) reading bitstream data len",
+			modminer->api->name, modminer->device_id, errno);
+
+		goto dame;
+	}
+
 	len = ((unsigned long)ubuf[0] << 24) | ((unsigned long)ubuf[1] << 16) | (ubuf[2] << 8) | ubuf[3];
-	applog(LOG_DEBUG, "  Bitstream size: %lu", len);
+	applog(LOG_DEBUG, " Bitstream size: %lu", len);
+
+	strcpy(devmsg, modminer->device_path);
+	ptr = strrchr(devmsg, ':');
+	if (ptr)
+		*ptr = '\0';
 
-	SOCKETTYPE fd = modminer->device_fd;
+	applog(LOG_WARNING, "%s%u: Programming all FPGA on %s ... Mining will not start until complete",
+		modminer->api->name, modminer->device_id, devmsg);
 
-	applog(LOG_WARNING, "%s %u: Programming %s... DO NOT EXIT CGMINER UNTIL COMPLETE", modminer->api->name, modminer->device_id, modminer->device_path);
-	buf[0] = '\x05';  // Program Bitstream
+	buf[0] = MODMINER_PROGRAM;
 	buf[1] = fpgaid;
 	buf[2] = (len >>  0) & 0xff;
 	buf[3] = (len >>  8) & 0xff;
 	buf[4] = (len >> 16) & 0xff;
 	buf[5] = (len >> 24) & 0xff;
-	if (6 != write(fd, buf, 6))
-		bailout2(LOG_ERR, "%s %u: Error programming %s (cmd)", modminer->api->name, modminer->device_id, modminer->device_path);
-	status_read("cmd reply");
-	ssize_t buflen;
-	while (len) {
-		buflen = len < 32 ? len : 32;
-		if (fread(buf, buflen, 1, f) != 1)
-			bailout2(LOG_ERR, "%s %u: File underrun programming %s (%d bytes left)", modminer->api->name, modminer->device_id, modminer->device_path, len);
-		if (write(fd, buf, buflen) != buflen)
-			bailout2(LOG_ERR, "%s %u: Error programming %s (data)", modminer->api->name, modminer->device_id,  modminer->device_path);
-		status_read("status");
-		len -= buflen;
+
+	if ((err = usb_write(modminer, buf, 6, &amount, C_STARTPROGRAM)) < 0 || amount != 6) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Program init failed (%d:%d)",
+			modminer->api->name, modminer->device_id, amount, err);
+
+		goto dame;
 	}
-	status_read("final status");
-	applog(LOG_WARNING, "%s %u: Done programming %s", modminer->api->name, modminer->device_id, modminer->device_path);
 
-	return true;
-}
+	if (!get_status(modminer, "initialise", C_STARTPROGRAMSTATUS))
+		goto undame;
 
-static bool modminer_device_prepare(struct cgpu_info *modminer)
-{
-	int fd = serial_open(modminer->device_path, 0, 10, true);
-	if (unlikely(-1 == fd))
-		bailout(LOG_ERR, "%s %u: Failed to open %s", modminer->api->name, modminer->device_id, modminer->device_path);
+// It must be 32 bytes according to MCU legacy.c
+#define WRITE_SIZE 32
 
-	modminer->device_fd = fd;
-	applog(LOG_INFO, "%s %u: Opened %s", modminer->api->name, modminer->device_id, modminer->device_path);
+	totlen = len;
+	nextmsg = 0.1;
+	while (len > 0) {
+		buflen = len < WRITE_SIZE ? len : WRITE_SIZE;
+		if (fread(buf, buflen, 1, f) != 1) {
+			mutex_unlock(modminer->modminer_mutex);
 
-	struct timeval now;
-	gettimeofday(&now, NULL);
-	get_datestamp(modminer->init, &now);
+			applog(LOG_ERR, "%s%u: bitstream file read error %d (%d bytes left)",
+				modminer->api->name, modminer->device_id, errno, len);
+
+			goto dame;
+		}
+
+		tries = 0;
+		ptr = buf;
+		remaining = buflen;
+		while ((err = usb_write(modminer, ptr, remaining, &amount, C_PROGRAM)) < 0 || amount != (int)remaining) {
+			if (err == LIBUSB_ERROR_TIMEOUT && amount > 0 && ++tries < 4) {
+				remaining -= amount;
+				ptr += amount;
+
+				if (opt_debug)
+					applog(LOG_DEBUG, "%s%u: Program timeout (%d:%d) sent %d tries %d",
+						modminer->api->name, modminer->device_id,
+						amount, err, remaining, tries);
+
+				if (!get_status(modminer, "write status", C_PROGRAMSTATUS2))
+					goto dame;
+
+			} else {
+				mutex_unlock(modminer->modminer_mutex);
+
+				applog(LOG_ERR, "%s%u: Program failed (%d:%d) sent %d",
+					modminer->api->name, modminer->device_id, amount, err, remaining);
+
+				goto dame;
+			}
+		}
+
+		if (!get_status(modminer, "write status", C_PROGRAMSTATUS))
+			goto dame;
+
+		len -= buflen;
+
+		upto = (float)(totlen - len) / (float)(totlen);
+		if (upto >= nextmsg) {
+			applog(LOG_WARNING,
+				"%s%u: Programming %.1f%% (%d out of %d)",
+				modminer->api->name, modminer->device_id, upto*100, (totlen - len), totlen);
+
+			nextmsg += 0.1;
+		}
+	}
+
+	if (!get_status(modminer, "final status", C_FINALPROGRAMSTATUS))
+		goto undame;
+
+	applog(LOG_WARNING, "%s%u: Programming completed for all FPGA on %s",
+		modminer->api->name, modminer->device_id, devmsg);
+
+	// Give it a 2/3s delay after programming
+	nmsleep(666);
 
 	return true;
+undame:
+	;
+	mutex_unlock(modminer->modminer_mutex);
+	;
+dame:
+	fclose(f);
+	return false;
 }
 
-#undef bailout
-
 static bool modminer_fpga_prepare(struct thr_info *thr)
 {
 	struct cgpu_info *modminer = thr->cgpu;
+	struct timeval now;
 
-	// Don't need to lock the mutex here,
-	// since prepare runs from the main thread before the miner threads start
-	if (modminer->device_fd == -1 && !modminer_device_prepare(modminer))
-		return false;
+	gettimeofday(&now, NULL);
+	get_datestamp(modminer->init, &now);
 
 	struct modminer_fpga_state *state;
 	state = thr->cgpu_data = calloc(1, sizeof(struct modminer_fpga_state));
-	state->next_work_cmd[0] = '\x08';  // Send Job
-	state->next_work_cmd[1] = thr->device_thread;  // FPGA id
+	state->next_work_cmd[0] = MODMINER_SEND_WORK;
+	state->next_work_cmd[1] = modminer->fpgaid;
 	state->shares_to_good = MODMINER_EARLY_UP;
+	state->overheated = false;
 
 	return true;
 }
 
 /*
  * Clocking rules:
- *	If device exceeds cutoff temp - shut down - and decrease the clock by
- *		MODMINER_OVERHEAT_CLOCK for when it restarts
+ *	If device exceeds cutoff temp - TODO: ?stop sending work -
+ *		and decrease the clock by MODMINER_OVERHEAT_CLOCK
+ *		for when it restarts
  *
  * When to clock down:
  *	If device overheats
+ *		also halve shares_to_good
+ *		(so multiple temp drops can recover faster)
  *	 or
  *	If device gets MODMINER_HW_ERROR_PERCENT errors since last clock up or down
  *		if clock is <= default it requires 2 HW to do this test
  *		if clock is > default it only requires 1 HW to do this test
+ *			also double shares_to_good
  *
  * When to clock up:
  *	If device gets shares_to_good good shares in a row
+ *		and temp <= MODMINER_TEMP_UP_LIMIT
  *
  * N.B. clock must always be a multiple of 2
  */
-static bool modminer_delta_clock(struct thr_info *thr, bool needlock, int delta, bool temp)
+static bool modminer_delta_clock(struct thr_info *thr, int delta, bool temp)
 {
 	struct cgpu_info *modminer = thr->cgpu;
 	struct modminer_fpga_state *state = thr->cgpu_data;
-	char fpgaid = thr->device_thread;
-	int fd = modminer->device_fd;
 	unsigned char cmd[6], buf[1];
-	struct timeval now;
-
-	gettimeofday(&now, NULL);
+	int err, amount;
 
 	// Only do once if multiple shares per work or multiple reasons
 	// Since the temperature down clock test is first in the code this is OK
-	if (tdiff(&now, &(state->last_changed)) < 0.5)
+	if (!state->new_work)
 		return false;
 
-	// Update before possibly aborting to avoid repeating unnecessarily
-	memcpy(&(state->last_changed), &now, sizeof(struct timeval));
+	state->new_work = false;
+
 	state->shares = 0;
 	state->shares_last_hw = 0;
 	state->hw_errors = 0;
 
 	// If drop requested due to temperature, clock drop is always allowed
-	if (!temp && delta < 0 && state->clock <= MODMINER_MIN_CLOCK)
+	if (!temp && delta < 0 && modminer->clock <= MODMINER_MIN_CLOCK)
 		return false;
 
-	if (delta > 0 && state->clock >= MODMINER_MAX_CLOCK)
+	if (delta > 0 && modminer->clock >= MODMINER_MAX_CLOCK)
 		return false;
 
 	if (delta < 0) {
-		if ((state->shares_to_good * 2) < MODMINER_TRY_UP)
-			state->shares_to_good *= 2;
-		else
-			state->shares_to_good = MODMINER_TRY_UP;
+		if (temp) {
+			if (state->shares_to_good > MODMINER_MIN_BACK)
+				state->shares_to_good /= 2;
+		} else {
+			if ((state->shares_to_good * 2) < MODMINER_TRY_UP)
+				state->shares_to_good *= 2;
+			else
+				state->shares_to_good = MODMINER_TRY_UP;
+		}
 	}
 
-	state->clock += delta;
+	modminer->clock += delta;
 
-	cmd[0] = '\x06';  // set clock speed
-	cmd[1] = fpgaid;
-	cmd[2] = state->clock;
+	cmd[0] = MODMINER_SET_CLOCK;
+	cmd[1] = modminer->fpgaid;
+	cmd[2] = modminer->clock;
 	cmd[3] = cmd[4] = cmd[5] = '\0';
 
-	if (needlock)
-		mutex_lock(&modminer->device_mutex);
-	if (6 != write(fd, cmd, 6))
-		bailout2(LOG_ERR, "%s%u.%u: Error writing (set clock speed)", modminer->api->name, modminer->device_id, fpgaid);
-	if (serial_read(fd, &buf, 1) != 1)
-		bailout2(LOG_ERR, "%s%u.%u: Error reading (set clock speed)", modminer->api->name, modminer->device_id, fpgaid);
-	if (needlock)
-		mutex_unlock(&modminer->device_mutex);
+	mutex_lock(modminer->modminer_mutex);
+
+	if ((err = usb_write(modminer, (char *)cmd, 6, &amount, C_SETCLOCK)) < 0 || amount != 6) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Error writing set clock speed (%d:%d)",
+			modminer->api->name, modminer->device_id, amount, err);
+
+		return false;
+	}
+
+	if ((err = usb_read(modminer, (char *)(&buf), 1, &amount, C_REPLYSETCLOCK)) < 0 || amount != 1) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Error reading set clock speed (%d:%d)",
+			modminer->api->name, modminer->device_id, amount, err);
 
-	applog(LOG_WARNING, "%s%u.%u: Set clock speed %sto %u", modminer->api->name, modminer->device_id, fpgaid, (delta < 0) ? "down " : (delta > 0 ? "up " : ""), state->clock);
+		return false;
+	}
+
+	mutex_unlock(modminer->modminer_mutex);
+
+	applog(LOG_WARNING, "%s%u: Set clock speed %sto %u",
+			modminer->api->name, modminer->device_id,
+			(delta < 0) ? "down " : (delta > 0 ? "up " : ""),
+			modminer->clock);
 
 	return true;
 }
@@ -424,39 +665,48 @@ static bool modminer_delta_clock(struct thr_info *thr, bool needlock, int delta,
 static bool modminer_fpga_init(struct thr_info *thr)
 {
 	struct cgpu_info *modminer = thr->cgpu;
-	struct modminer_fpga_state *state = thr->cgpu_data;
-	int fd;
-	char fpgaid = thr->device_thread;
-
 	unsigned char cmd[2], buf[4];
+	int err, amount;
+
+	mutex_lock(modminer->modminer_mutex);
+
+	cmd[0] = MODMINER_GET_USERCODE;
+	cmd[1] = modminer->fpgaid;
+	if ((err = usb_write(modminer, (char *)cmd, 2, &amount, C_REQUESTUSERCODE)) < 0 || amount != 2) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Error requesting USER code (%d:%d)",
+			modminer->api->name, modminer->device_id, amount, err);
 
-	mutex_lock(&modminer->device_mutex);
-	fd = modminer->device_fd;
-	if (fd == -1) {
-		// Died in another thread...
-		mutex_unlock(&modminer->device_mutex);
 		return false;
 	}
 
-	cmd[0] = '\x04';  // Read USER code (bitstream id)
-	cmd[1] = fpgaid;
-	if (write(fd, cmd, 2) != 2)
-		bailout2(LOG_ERR, "%s%u.%u: Error writing (read USER code)", modminer->api->name, modminer->device_id, fpgaid);
-	if (serial_read(fd, buf, 4) != 4)
-		bailout2(LOG_ERR, "%s%u.%u: Error reading (read USER code)", modminer->api->name, modminer->device_id, fpgaid);
+	if ((err = usb_read(modminer, (char *)buf, 4, &amount, C_GETUSERCODE)) < 0 || amount != 4) {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Error reading USER code (%d:%d)",
+			modminer->api->name, modminer->device_id, amount, err);
+
+		return false;
+	}
 
 	if (memcmp(buf, BISTREAM_USER_ID, 4)) {
-		applog(LOG_ERR, "%s%u.%u: FPGA not programmed", modminer->api->name, modminer->device_id, fpgaid);
+		applog(LOG_ERR, "%s%u: FPGA not programmed",
+			modminer->api->name, modminer->device_id);
+
 		if (!modminer_fpga_upload_bitstream(modminer))
 			return false;
-	}
-	else
-		applog(LOG_DEBUG, "%s%u.%u: FPGA is already programmed :)", modminer->api->name, modminer->device_id, fpgaid);
 
-	state->clock = MODMINER_DEF_CLOCK;
-	modminer_delta_clock(thr, false, MODMINER_CLOCK_SET, false);
+		mutex_unlock(modminer->modminer_mutex);
+	} else {
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_DEBUG, "%s%u: FPGA is already programmed :)",
+			modminer->api->name, modminer->device_id);
+	}
 
-	mutex_unlock(&modminer->device_mutex);
+	modminer->clock = MODMINER_DEF_CLOCK;
+	modminer_delta_clock(thr, MODMINER_CLOCK_SET, false);
 
 	thr->primary_thread = true;
 
@@ -465,34 +715,14 @@ static bool modminer_fpga_init(struct thr_info *thr)
 
 static void get_modminer_statline_before(char *buf, struct cgpu_info *modminer)
 {
-	char info[18] = "               | ";
-	int tc = modminer->threads;
-	bool havetemp = false;
-	int i;
+	char info[18];
 
-	if (tc > 4)
-		tc = 4;
+	sprintf(info, " %s%.1fC %3uMHz  | ",
+			(modminer->temp < 10) ? " " : "",
+			modminer->temp,
+			(unsigned int)(modminer->clock));
 
-	for (i = tc - 1; i >= 0; --i) {
-		struct thr_info *thr = modminer->thr[i];
-		struct modminer_fpga_state *state = thr->cgpu_data;
-		float temp = state->temp;
-
-		info[i*3+2] = '/';
-		if (temp) {
-			havetemp = true;
-			if (temp > 9)
-				info[i*3+0] = 0x30 + (temp / 10);
-			info[i*3+1] = 0x30 + ((int)temp % 10);
-		}
-	}
-	if (havetemp) {
-		info[tc*3-1] = ' ';
-		info[tc*3] = 'C';
-		strcat(buf, info);
-	}
-	else
-		strcat(buf, "               | ");
+	strcat(buf, info);
 }
 
 static bool modminer_prepare_next_work(struct modminer_fpga_state *state, struct work *work)
@@ -508,76 +738,165 @@ static bool modminer_prepare_next_work(struct modminer_fpga_state *state, struct
 
 static bool modminer_start_work(struct thr_info *thr)
 {
-fd_set fds;
 	struct cgpu_info *modminer = thr->cgpu;
 	struct modminer_fpga_state *state = thr->cgpu_data;
-	char fpgaid = thr->device_thread;
-	SOCKETTYPE fd = modminer->device_fd;
+	int err, amount;
+	bool sta;
 
-	char buf[1];
+	mutex_lock(modminer->modminer_mutex);
+
+	if ((err = usb_write(modminer, (char *)(state->next_work_cmd), 46, &amount, C_SENDWORK)) < 0 || amount != 46) {
+// TODO: err = -4 means the MMQ disappeared - need to delete it and rescan for it? (after a delay?)
+// but check all (4) disappeared
+		mutex_unlock(modminer->modminer_mutex);
+
+		applog(LOG_ERR, "%s%u: Start work failed (%d:%d)",
+			modminer->api->name, modminer->device_id, amount, err);
+
+		return false;
+	}
 
-	mutex_lock(&modminer->device_mutex);
-	if (46 != write(fd, state->next_work_cmd, 46))
-		bailout2(LOG_ERR, "%s%u.%u: Error writing (start work)", modminer->api->name, modminer->device_id, fpgaid);
 	gettimeofday(&state->tv_workstart, NULL);
 	state->hashes = 0;
-	status_read("start work");
-	mutex_unlock(&modminer->device_mutex);
 
-	return true;
-}
+	sta = get_status(modminer, "start work", C_SENDWORKSTATUS);
 
-#define work_restart(thr)  thr->work_restart
+	if (sta) {
+		mutex_unlock(modminer->modminer_mutex);
+		state->new_work = true;
+	}
 
-static uint64_t modminer_process_results(struct thr_info *thr)
+	return sta;
+}
+
+static void check_temperature(struct thr_info *thr)
 {
 	struct cgpu_info *modminer = thr->cgpu;
 	struct modminer_fpga_state *state = thr->cgpu_data;
-	char fpgaid = thr->device_thread;
-	int fd = modminer->device_fd;
-	struct work *work = &state->running_work;
-
 	char cmd[2], temperature[2];
-	uint32_t nonce;
-	long iter;
-	uint32_t curr_hw_errors;
+	int tbytes, tamount;
+	int amount;
+
+	if (modminer->one_byte_temp) {
+		cmd[0] = MODMINER_TEMP1;
+		tbytes = 1;
+	} else {
+		cmd[0] = MODMINER_TEMP2;
+		tbytes = 2;
+	}
 
-	// \x0a is 1 byte temperature
-	// \x0d is 2 byte temperature
-	cmd[0] = '\x0d';
-	cmd[1] = fpgaid;
+	cmd[1] = modminer->fpgaid;
 
-	mutex_lock(&modminer->device_mutex);
-	if (2 == write(fd, cmd, 2) && read(fd, &temperature, 2) == 2)
+	mutex_lock(modminer->modminer_mutex);
+	if (usb_write(modminer, (char *)cmd, 2, &amount, C_REQUESTTEMPERATURE) == 0 && amount == 2
+	&&  usb_read(modminer, (char *)(&temperature), tbytes, &tamount, C_GETTEMPERATURE) == 0 && tamount == tbytes)
 	{
-		// Only accurate to 2 and a bit places
-		state->temp = roundf((temperature[1] * 256.0 + temperature[0]) / 0.128) / 1000.0;
-		if (!fpgaid)
-			modminer->temp = state->temp;
-
-		if (state->temp >= MODMINER_OVERHEAT_TEMP) {
-			if (state->temp >= MODMINER_CUTOFF_TEMP) {
-				applog(LOG_WARNING, "%s%u.%u: Hit thermal cutoff limit (%f) at %f, disabling device!", modminer->api->name, modminer->device_id, fpgaid, MODMINER_CUTOFF_TEMP, state->temp);
-				modminer_delta_clock(thr, true, MODMINER_OVERHEAT_CLOCK, true);
+		mutex_unlock(modminer->modminer_mutex);
+		if (modminer->one_byte_temp)
+			modminer->temp = temperature[0];
+		else {
+			// Only accurate to 2 and a bit places
+			modminer->temp = roundf((temperature[1] * 256.0 + temperature[0]) / 0.128) / 1000.0;
+
+			modminer->tried_two_byte_temp = true;
+		}
 
+		if (state->overheated) {
+			if (modminer->temp < MODMINER_OVERHEAT_TEMP) {
+				state->overheated = false;
+				applog(LOG_WARNING, "%s%u: Recovered, temp less than (%f) now %f",
+					modminer->api->name, modminer->device_id,
+					MODMINER_OVERHEAT_TEMP, modminer->temp);
+			}
+		}
+		else if (modminer->temp >= MODMINER_OVERHEAT_TEMP) {
+			if (modminer->temp >= MODMINER_CUTOFF_TEMP) {
+				applog(LOG_WARNING, "%s%u: Hit thermal cutoff limit (%f) at %f, disabling!",
+					modminer->api->name, modminer->device_id,
+					MODMINER_CUTOFF_TEMP, modminer->temp);
+
+				modminer_delta_clock(thr, MODMINER_OVERHEAT_CLOCK, true);
+				state->overheated = true;
 				dev_error(modminer, REASON_DEV_THERMAL_CUTOFF);
-				modminer->deven = DEV_RECOVER;
 			} else {
-				 applog(LOG_WARNING, "%s%u.%u Overheat limit (%f) reached %f", modminer->api->name, modminer->device_id, fpgaid, MODMINER_OVERHEAT_TEMP, state->temp);
-				modminer_delta_clock(thr, true, MODMINER_CLOCK_DOWN, true);
-
+				 applog(LOG_WARNING, "%s%u: Overheat limit (%f) reached %f",
+					modminer->api->name, modminer->device_id,
+					MODMINER_OVERHEAT_TEMP, modminer->temp);
+				modminer_delta_clock(thr, MODMINER_CLOCK_DOWN, true);
 				dev_error(modminer, REASON_DEV_OVER_HEAT);
 			}
 		}
+	} else {
+		mutex_unlock(modminer->modminer_mutex);
+
+		if (!modminer->tried_two_byte_temp) {
+			modminer->tried_two_byte_temp = true;
+			modminer->one_byte_temp = true;
+		}
 	}
+}
+
+#define work_restart(thr)  thr->work_restart
 
-	cmd[0] = '\x09';
+static uint64_t modminer_process_results(struct thr_info *thr)
+{
+	struct cgpu_info *modminer = thr->cgpu;
+	struct modminer_fpga_state *state = thr->cgpu_data;
+	struct work *work = &state->running_work;
+	char cmd[2];
+	uint32_t nonce;
+	long iter;
+	uint32_t curr_hw_errors;
+	int err, amount;
+	int timeoutloop;
+
+	check_temperature(thr);
+
+	if (state->overheated == true) {
+		if (state->work_running)
+			state->work_running = false;
+
+		// Give it 5 seconds rest and wait for the next work
+		nmsleep(5000);
+		return 0;
+	}
+
+	cmd[0] = MODMINER_CHECK_WORK;
+	cmd[1] = modminer->fpgaid;
 	iter = 200;
+	timeoutloop = 0;
 	while (1) {
-		if (write(fd, cmd, 2) != 2)
-			bailout2(LOG_ERR, "%s%u.%u: Error reading (get nonce)", modminer->api->name, modminer->device_id, fpgaid);
-		serial_read(fd, &nonce, 4);
-		mutex_unlock(&modminer->device_mutex);
+		mutex_lock(modminer->modminer_mutex);
+		if ((err = usb_write(modminer, cmd, 2, &amount, C_REQUESTWORKSTATUS)) < 0 || amount != 2) {
+// TODO: err = -4 means the MMQ disappeared - need to delete it and rescan for it? (after a delay?)
+// but check all (4) disappeared
+			mutex_unlock(modminer->modminer_mutex);
+
+			// timeoutloop never resets so the timeouts can't
+			// accumulate much during a single item of work
+			if (err == -7 && ++timeoutloop < 10)
+				goto tryagain;
+
+			applog(LOG_ERR, "%s%u: Error sending (get nonce) (%d:%d)",
+				modminer->api->name, modminer->device_id, amount, err);
+
+			return -1;
+		}
+
+		err = usb_read(modminer, (char *)(&nonce), 4, &amount, C_GETWORKSTATUS);
+		mutex_unlock(modminer->modminer_mutex);
+
+		if (err < 0 || amount != 4) {
+
+			// timeoutloop never resets so the timeouts can't
+			// accumulate much during a single item of work
+			if (err == -7 && ++timeoutloop < 10)
+				goto tryagain;
+
+			applog(LOG_ERR, "%s%u: Error reading (get nonce) (%d:%d)",
+				modminer->api->name, modminer->device_id, amount, err);
+		}
+
 		if (memcmp(&nonce, "\xff\xff\xff\xff", 4)) {
 			state->shares++;
 			state->no_nonce_counter = 0;
@@ -585,36 +904,41 @@ static uint64_t modminer_process_results(struct thr_info *thr)
 			submit_nonce(thr, work, nonce);
 			if (state->hw_errors > curr_hw_errors) {
 				state->shares_last_hw = state->shares;
-				if (state->clock > MODMINER_DEF_CLOCK || state->hw_errors > 1) {
+				if (modminer->clock > MODMINER_DEF_CLOCK || state->hw_errors > 1) {
 					float pct = (state->hw_errors * 100.0 / (state->shares ? : 1.0));
 					if (pct >= MODMINER_HW_ERROR_PERCENT)
-						modminer_delta_clock(thr, true, MODMINER_CLOCK_DOWN, false);
+						modminer_delta_clock(thr, MODMINER_CLOCK_DOWN, false);
 				}
 			} else {
 				// If we've reached the required good shares in a row then clock up
-				if ((state->shares - state->shares_last_hw) >= state->shares_to_good)
-					modminer_delta_clock(thr, true, MODMINER_CLOCK_UP, false);
+				if (((state->shares - state->shares_last_hw) >= state->shares_to_good) &&
+					modminer->temp <= MODMINER_TEMP_UP_LIMIT)
+					modminer_delta_clock(thr, MODMINER_CLOCK_UP, false);
 			}
 		} else if (++state->no_nonce_counter > 18000) {
-			// TODO: NFI what this is - but will be gone
-			// when the threading rewrite is done
+			// TODO: NFI what this is
 			state->no_nonce_counter = 0;
-			modminer_delta_clock(thr, true, MODMINER_CLOCK_DOWN, false);
+			modminer_delta_clock(thr, MODMINER_CLOCK_DOWN, false);
+
+			applog(LOG_ERR, "%s%u: 18000 clock down",
+				modminer->api->name, modminer->device_id);
+
 		}
 
+tryagain:
+
 		if (work_restart(thr))
 			break;
 		nmsleep(10);
 		if (work_restart(thr) || !--iter)
 			break;
-		mutex_lock(&modminer->device_mutex);
 	}
 
 	struct timeval tv_workend, elapsed;
 	gettimeofday(&tv_workend, NULL);
 	timersub(&tv_workend, &state->tv_workstart, &elapsed);
 
-	uint64_t hashes = (uint64_t)state->clock * (((uint64_t)elapsed.tv_sec * 1000000) + elapsed.tv_usec);
+	uint64_t hashes = (uint64_t)modminer->clock * (((uint64_t)elapsed.tv_sec * 1000000) + elapsed.tv_usec);
 	if (hashes > 0xffffffff)
 		hashes = 0xffffffff;
 	else
@@ -632,9 +956,25 @@ static int64_t modminer_scanhash(struct thr_info *thr, struct work *work, int64_
 	int64_t hashes = 0;
 	bool startwork;
 
+	if (state->overheated == true) {
+		if (state->work_running)
+			state->work_running = false;
+
+		check_temperature(thr);
+
+		if (state->overheated == true) {
+			// Give it 5 seconds rest and wait for the next work
+			nmsleep(5000);
+			return 0;
+		}
+	}
+
 	startwork = modminer_prepare_next_work(state, work);
 	if (state->work_running) {
 		hashes = modminer_process_results(thr);
+		if (hashes == -1)
+			return hashes;
+
 		if (work_restart(thr)) {
 			state->work_running = false;
 			return 0;
diff --git a/miner.h b/miner.h
index 820ef63..6c38cf0 100644
--- a/miner.h
+++ b/miner.h
@@ -110,6 +110,10 @@ static inline int fsync (int fd)
   #include "libztex.h"
 #endif
 
+#ifdef USE_MODMINER
+  #include "usbutils.h"
+#endif
+
 #if !defined(WIN32) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
 #define bswap_16 __builtin_bswap16
 #define bswap_32 __builtin_bswap32
@@ -361,8 +365,19 @@ struct cgpu_info {
 #ifdef USE_ZTEX
 		struct libztex_device *device_ztex;
 #endif
+#ifdef USE_MODMINER
+		struct cg_usb_device *usbdev;
+#endif
 		int device_fd;
 	};
+#ifdef USE_MODMINER
+	int usbstat;
+	char fpgaid;
+	unsigned char clock;
+	pthread_mutex_t *modminer_mutex;
+	bool tried_two_byte_temp;
+	bool one_byte_temp;
+#endif
 #ifdef USE_BITFORCE
 	struct timeval work_start_tv;
 	unsigned int wait_ms;
@@ -373,9 +388,8 @@ struct cgpu_info {
 	bool nonce_range;
 	bool polling;
 	bool flash_led;
+	pthread_mutex_t device_mutex;
 #endif
-	pthread_mutex_t		device_mutex;
-
 	enum dev_enable deven;
 	int accepted;
 	int rejected;
@@ -654,6 +668,9 @@ extern bool opt_restart;
 extern char *opt_icarus_options;
 extern char *opt_icarus_timing;
 extern bool opt_worktime;
+#ifdef HAVE_LIBUSB
+extern int opt_usbdump;
+#endif
 #ifdef USE_BITFORCE
 extern bool opt_bfl_noncerange;
 #endif
@@ -684,6 +701,10 @@ extern int opt_queue;
 extern int opt_scantime;
 extern int opt_expiry;
 
+#ifdef HAVE_LIBUSB
+extern pthread_mutex_t cgusb_lock;
+#endif
+
 extern pthread_mutex_t console_lock;
 extern pthread_mutex_t ch_lock;
 
@@ -998,9 +1019,10 @@ struct modminer_fpga_state {
 	uint32_t hashes;
 
 	char next_work_cmd[46];
+	char fpgaid;
 
-	unsigned char clock;
-	float temp;
+	bool overheated;
+	bool new_work;
 
 	uint32_t shares;
 	uint32_t shares_last_hw;
diff --git a/usbutils.c b/usbutils.c
new file mode 100644
index 0000000..36beb30
--- /dev/null
+++ b/usbutils.c
@@ -0,0 +1,1192 @@
+/*
+ * Copyright 2012 Andrew Smith
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option)
+ * any later version.  See COPYING for more details.
+ */
+
+#include "config.h"
+
+#include <stdint.h>
+#include <stdbool.h>
+
+#include "logging.h"
+#include "miner.h"
+#include "usbutils.h"
+
+#ifdef USE_ICARUS
+#define DRV_ICARUS 1
+#endif
+
+#ifdef USE_BITFORCE
+#define DRV_BITFORCE 2
+#endif
+
+#ifdef USE_MODMINER
+#define DRV_MODMINER 3
+#endif
+
+#define DRV_LAST -1
+
+#define USB_CONFIG 1
+
+#define EPI(x) (LIBUSB_ENDPOINT_IN | (unsigned char)(x))
+#define EPO(x) (LIBUSB_ENDPOINT_OUT | (unsigned char)(x))
+
+#ifdef WIN32
+#define MODMINER_TIMEOUT_MS 200
+#else
+#define MODMINER_TIMEOUT_MS 100
+#endif
+
+#ifdef USE_MODMINER
+static struct usb_endpoints mmq_eps[] = {
+	{ LIBUSB_TRANSFER_TYPE_BULK,	64,	EPI(3), 0 },
+	{ LIBUSB_TRANSFER_TYPE_BULK,	64,	EPO(3), 0 }
+};
+#endif
+
+// TODO: Add support for (at least) Interrupt endpoints
+static struct usb_find_devices find_dev[] = {
+/*
+#ifdef USE_ICARUS
+	{ DRV_ICARUS, 	"ICA",	0x067b,	0x0230,	true,	EPI(3),	EPO(2), 1 },
+	{ DRV_ICARUS, 	"LOT",	0x0403,	0x6001,	false,	EPI(0),	EPO(0), 1 },
+	{ DRV_ICARUS, 	"CM1",	0x067b,	0x0230,	false,	EPI(0),	EPO(0), 1 },
+#endif
+#ifdef USE_BITFORCE
+	{ DRV_BITFORCE,	"BFL",	0x0403,	0x6014,	true,	EPI(1),	EPO(2), 1 },
+#endif
+*/
+#ifdef USE_MODMINER
+	{
+		.drv = DRV_MODMINER,
+		.name = "MMQ",
+		.idVendor = 0x1fc9,
+		.idProduct = 0x0003,
+		.config = 1,
+		.interface = 1,
+		.timeout = MODMINER_TIMEOUT_MS,
+		.epcount = ARRAY_SIZE(mmq_eps),
+		.eps = mmq_eps },
+#endif
+	{ DRV_LAST, NULL, 0, 0, 0, 0, 0, 0, NULL }
+};
+
+#ifdef USE_BITFORCE
+extern struct device_api bitforce_api;
+#endif
+
+#ifdef USE_ICARUS
+extern struct device_api icarus_api;
+#endif
+
+#ifdef USE_MODMINER
+extern struct device_api modminer_api;
+#endif
+
+/*
+ * Our own internal list of used USB devices
+ * So two drivers or a single driver searching
+ * can't touch the same device during detection
+ */
+struct usb_list {
+	uint8_t bus_number;
+	uint8_t device_address;
+	uint8_t filler[2];
+	struct usb_list *prev;
+	struct usb_list *next;
+};
+
+#define STRBUFLEN 256
+static const char *BLANK = "";
+
+static pthread_mutex_t *list_lock = NULL;
+static struct usb_list *usb_head = NULL;
+
+struct cg_usb_stats_item {
+	uint64_t count;
+	double total_delay;
+	double min_delay;
+	double max_delay;
+	struct timeval first;
+	struct timeval last;
+};
+
+#define CMD_CMD 0
+#define CMD_TIMEOUT 1
+#define CMD_ERROR 2
+
+struct cg_usb_stats_details {
+	int seq;
+	struct cg_usb_stats_item item[CMD_ERROR+1];
+};
+
+struct cg_usb_stats {
+	char *name;
+	int device_id;
+	struct cg_usb_stats_details *details;
+};
+
+#define SEQ0 0
+#define SEQ1 1
+
+static struct cg_usb_stats *usb_stats = NULL;
+static int next_stat = 0;
+
+static const char **usb_commands;
+
+static const char *C_PING_S = "Ping";
+static const char *C_CLEAR_S = "Clear";
+static const char *C_REQUESTVERSION_S = "RequestVersion";
+static const char *C_GETVERSION_S = "GetVersion";
+static const char *C_REQUESTFPGACOUNT_S = "RequestFPGACount";
+static const char *C_GETFPGACOUNT_S = "GetFPGACount";
+static const char *C_STARTPROGRAM_S = "StartProgram";
+static const char *C_STARTPROGRAMSTATUS_S = "StartProgramStatus";
+static const char *C_PROGRAM_S = "Program";
+static const char *C_PROGRAMSTATUS_S = "ProgramStatus";
+static const char *C_PROGRAMSTATUS2_S = "ProgramStatus2";
+static const char *C_FINALPROGRAMSTATUS_S = "FinalProgramStatus";
+static const char *C_SETCLOCK_S = "SetClock";
+static const char *C_REPLYSETCLOCK_S = "ReplySetClock";
+static const char *C_REQUESTUSERCODE_S = "RequestUserCode";
+static const char *C_GETUSERCODE_S = "GetUserCode";
+static const char *C_REQUESTTEMPERATURE_S = "RequestTemperature";
+static const char *C_GETTEMPERATURE_S = "GetTemperature";
+static const char *C_SENDWORK_S = "SendWork";
+static const char *C_SENDWORKSTATUS_S = "SendWorkStatus";
+static const char *C_REQUESTWORKSTATUS_S = "RequestWorkStatus";
+static const char *C_GETWORKSTATUS_S = "GetWorkStatus";
+
+#ifdef EOL
+#undef EOL
+#endif
+#define EOL "\n"
+
+static const char *DESDEV = "Device";
+static const char *DESCON = "Config";
+static const char *DESSTR = "String";
+static const char *DESINT = "Interface";
+static const char *DESEP = "Endpoint";
+static const char *DESHID = "HID";
+static const char *DESRPT = "Report";
+static const char *DESPHY = "Physical";
+static const char *DESHUB = "Hub";
+
+static const char *EPIN = "In: ";
+static const char *EPOUT = "Out: ";
+static const char *EPX = "?: ";
+
+static const char *CONTROL = "Control";
+static const char *ISOCHRONOUS_X = "Isochronous+?";
+static const char *ISOCHRONOUS_N_X = "Isochronous+None+?";
+static const char *ISOCHRONOUS_N_D = "Isochronous+None+Data";
+static const char *ISOCHRONOUS_N_F = "Isochronous+None+Feedback";
+static const char *ISOCHRONOUS_N_I = "Isochronous+None+Implicit";
+static const char *ISOCHRONOUS_A_X = "Isochronous+Async+?";
+static const char *ISOCHRONOUS_A_D = "Isochronous+Async+Data";
+static const char *ISOCHRONOUS_A_F = "Isochronous+Async+Feedback";
+static const char *ISOCHRONOUS_A_I = "Isochronous+Async+Implicit";
+static const char *ISOCHRONOUS_D_X = "Isochronous+Adaptive+?";
+static const char *ISOCHRONOUS_D_D = "Isochronous+Adaptive+Data";
+static const char *ISOCHRONOUS_D_F = "Isochronous+Adaptive+Feedback";
+static const char *ISOCHRONOUS_D_I = "Isochronous+Adaptive+Implicit";
+static const char *ISOCHRONOUS_S_X = "Isochronous+Sync+?";
+static const char *ISOCHRONOUS_S_D = "Isochronous+Sync+Data";
+static const char *ISOCHRONOUS_S_F = "Isochronous+Sync+Feedback";
+static const char *ISOCHRONOUS_S_I = "Isochronous+Sync+Implicit";
+static const char *BULK = "Bulk";
+static const char *INTERRUPT = "Interrupt";
+static const char *UNKNOWN = "Unknown";
+
+static const char *destype(uint8_t bDescriptorType)
+{
+	switch (bDescriptorType) {
+		case LIBUSB_DT_DEVICE:
+			return DESDEV;
+		case LIBUSB_DT_CONFIG:
+			return DESCON;
+		case LIBUSB_DT_STRING:
+			return DESSTR;
+		case LIBUSB_DT_INTERFACE:
+			return DESINT;
+		case LIBUSB_DT_ENDPOINT:
+			return DESEP;
+		case LIBUSB_DT_HID:
+			return DESHID;
+		case LIBUSB_DT_REPORT:
+			return DESRPT;
+		case LIBUSB_DT_PHYSICAL:
+			return DESPHY;
+		case LIBUSB_DT_HUB:
+			return DESHUB;
+	}
+	return UNKNOWN;
+}
+
+static const char *epdir(uint8_t bEndpointAddress)
+{
+	switch (bEndpointAddress & LIBUSB_ENDPOINT_DIR_MASK) {
+		case LIBUSB_ENDPOINT_IN:
+			return EPIN;
+		case LIBUSB_ENDPOINT_OUT:
+			return EPOUT;
+	}
+	return EPX;
+}
+
+static const char *epatt(uint8_t bmAttributes)
+{
+	switch(bmAttributes & LIBUSB_TRANSFER_TYPE_MASK) {
+		case LIBUSB_TRANSFER_TYPE_CONTROL:
+			return CONTROL;
+		case LIBUSB_TRANSFER_TYPE_BULK:
+			return BULK;
+		case LIBUSB_TRANSFER_TYPE_INTERRUPT:
+			return INTERRUPT;
+		case LIBUSB_TRANSFER_TYPE_ISOCHRONOUS:
+			switch(bmAttributes & LIBUSB_ISO_SYNC_TYPE_MASK) {
+				case LIBUSB_ISO_SYNC_TYPE_NONE:
+					switch(bmAttributes & LIBUSB_ISO_USAGE_TYPE_MASK) {
+						case LIBUSB_ISO_USAGE_TYPE_DATA:
+							return ISOCHRONOUS_N_D;
+						case LIBUSB_ISO_USAGE_TYPE_FEEDBACK:
+							return ISOCHRONOUS_N_F;
+						case LIBUSB_ISO_USAGE_TYPE_IMPLICIT:
+							return ISOCHRONOUS_N_I;
+					}
+					return ISOCHRONOUS_N_X;
+				case LIBUSB_ISO_SYNC_TYPE_ASYNC:
+					switch(bmAttributes & LIBUSB_ISO_USAGE_TYPE_MASK) {
+						case LIBUSB_ISO_USAGE_TYPE_DATA:
+							return ISOCHRONOUS_A_D;
+						case LIBUSB_ISO_USAGE_TYPE_FEEDBACK:
+							return ISOCHRONOUS_A_F;
+						case LIBUSB_ISO_USAGE_TYPE_IMPLICIT:
+							return ISOCHRONOUS_A_I;
+					}
+					return ISOCHRONOUS_A_X;
+				case LIBUSB_ISO_SYNC_TYPE_ADAPTIVE:
+					switch(bmAttributes & LIBUSB_ISO_USAGE_TYPE_MASK) {
+						case LIBUSB_ISO_USAGE_TYPE_DATA:
+							return ISOCHRONOUS_D_D;
+						case LIBUSB_ISO_USAGE_TYPE_FEEDBACK:
+							return ISOCHRONOUS_D_F;
+						case LIBUSB_ISO_USAGE_TYPE_IMPLICIT:
+							return ISOCHRONOUS_D_I;
+					}
+					return ISOCHRONOUS_D_X;
+				case LIBUSB_ISO_SYNC_TYPE_SYNC:
+					switch(bmAttributes & LIBUSB_ISO_USAGE_TYPE_MASK) {
+						case LIBUSB_ISO_USAGE_TYPE_DATA:
+							return ISOCHRONOUS_S_D;
+						case LIBUSB_ISO_USAGE_TYPE_FEEDBACK:
+							return ISOCHRONOUS_S_F;
+						case LIBUSB_ISO_USAGE_TYPE_IMPLICIT:
+							return ISOCHRONOUS_S_I;
+					}
+					return ISOCHRONOUS_S_X;
+			}
+			return ISOCHRONOUS_X;
+	}
+
+	return UNKNOWN;
+}
+
+static void append(char **buf, char *append, size_t *off, size_t *len)
+{
+	int new = strlen(append);
+	if ((new + *off) >= *len)
+	{
+		*len *= 2;
+		*buf = realloc(*buf, *len);
+	}
+
+	strcpy(*buf + *off, append);
+	*off += new;
+}
+
+static bool setgetdes(ssize_t count, libusb_device *dev, struct libusb_device_handle *handle, struct libusb_config_descriptor **config, int cd, char **buf, size_t *off, size_t *len)
+{
+	char tmp[512];
+	int err;
+
+	err = libusb_set_configuration(handle, cd);
+	if (err) {
+		sprintf(tmp, EOL "  ** dev %d: Failed to set config descriptor to %d, err %d",
+				(int)count, cd, err);
+		append(buf, tmp, off, len);
+		return false;
+	}
+
+	err = libusb_get_active_config_descriptor(dev, config);
+	if (err) {
+		sprintf(tmp, EOL "  ** dev %d: Failed to get active config descriptor set to %d, err %d",
+				(int)count, cd, err);
+		append(buf, tmp, off, len);
+		return false;
+	}
+
+	sprintf(tmp, EOL "  ** dev %d: Set & Got active config descriptor to %d, err %d",
+			(int)count, cd, err);
+	append(buf, tmp, off, len);
+	return true;
+}
+
+static void usb_full(ssize_t count, libusb_device *dev, char **buf, size_t *off, size_t *len)
+{
+	struct libusb_device_descriptor desc;
+	struct libusb_device_handle *handle;
+	struct libusb_config_descriptor *config;
+	const struct libusb_interface_descriptor *idesc;
+	const struct libusb_endpoint_descriptor *epdesc;
+	unsigned char man[STRBUFLEN+1];
+	unsigned char prod[STRBUFLEN+1];
+	unsigned char ser[STRBUFLEN+1];
+	char tmp[512];
+	int err, i, j, k;
+
+	err = libusb_get_device_descriptor(dev, &desc);
+	if (err) {
+		sprintf(tmp, EOL ".USB dev %d: Failed to get descriptor, err %d",
+					(int)count, err);
+		append(buf, tmp, off, len);
+		return;
+	}
+
+	sprintf(tmp, EOL ".USB dev %d: Device Descriptor:" EOL "\tLength: %d" EOL
+			"\tDescriptor Type: %s" EOL "\tUSB: %04x" EOL "\tDeviceClass: %d" EOL
+			"\tDeviceSubClass: %d" EOL "\tDeviceProtocol: %d" EOL "\tMaxPacketSize0: %d" EOL
+			"\tidVendor: %04x" EOL "\tidProduct: %04x" EOL "\tDeviceRelease: %x" EOL
+			"\tNumConfigurations: %d",
+				(int)count, (int)(desc.bLength), destype(desc.bDescriptorType),
+				desc.bcdUSB, (int)(desc.bDeviceClass), (int)(desc.bDeviceSubClass),
+				(int)(desc.bDeviceProtocol), (int)(desc.bMaxPacketSize0),
+				desc.idVendor, desc.idProduct, desc.bcdDevice,
+				(int)(desc.bNumConfigurations));
+	append(buf, tmp, off, len);
+
+	err = libusb_open(dev, &handle);
+	if (err) {
+		sprintf(tmp, EOL "  ** dev %d: Failed to open, err %d", (int)count, err);
+		append(buf, tmp, off, len);
+		return;
+	}
+
+	if (libusb_kernel_driver_active(handle, 0) == 1) {
+		sprintf(tmp, EOL "   * dev %d: kernel attached", (int)count);
+		append(buf, tmp, off, len);
+	}
+
+	err = libusb_get_active_config_descriptor(dev, &config);
+	if (err) {
+		if (!setgetdes(count, dev, handle, &config, 1, buf, off, len)
+		&&  !setgetdes(count, dev, handle, &config, 0, buf, off, len)) {
+			libusb_close(handle);
+			sprintf(tmp, EOL "  ** dev %d: Failed to set config descriptor to %d or %d",
+					(int)count, 1, 0);
+			append(buf, tmp, off, len);
+			return;
+		}
+	}
+
+	sprintf(tmp, EOL "     dev %d: Active Config:" EOL "\tDescriptorType: %s" EOL
+			"\tNumInterfaces: %d" EOL "\tConfigurationValue: %d" EOL
+			"\tAttributes: %d" EOL "\tMaxPower: %d",
+				(int)count, destype(config->bDescriptorType),
+				(int)(config->bNumInterfaces), (int)(config->iConfiguration),
+				(int)(config->bmAttributes), (int)(config->MaxPower));
+	append(buf, tmp, off, len);
+
+	for (i = 0; i < (int)(config->bNumInterfaces); i++) {
+		for (j = 0; j < config->interface[i].num_altsetting; j++) {
+			idesc = &(config->interface[i].altsetting[j]);
+
+			sprintf(tmp, EOL "     _dev %d: Interface Descriptor %d:" EOL
+					"\tDescriptorType: %s" EOL "\tInterfaceNumber: %d" EOL
+					"\tNumEndpoints: %d" EOL "\tInterfaceClass: %d" EOL
+					"\tInterfaceSubClass: %d" EOL "\tInterfaceProtocol: %d",
+						(int)count, j, destype(idesc->bDescriptorType),
+						(int)(idesc->bInterfaceNumber),
+						(int)(idesc->bNumEndpoints),
+						(int)(idesc->bInterfaceClass),
+						(int)(idesc->bInterfaceSubClass),
+						(int)(idesc->bInterfaceProtocol));
+			append(buf, tmp, off, len);
+
+			for (k = 0; k < (int)(idesc->bNumEndpoints); k++) {
+				epdesc = &(idesc->endpoint[k]);
+
+				sprintf(tmp, EOL "     __dev %d: Interface %d Endpoint %d:" EOL
+						"\tDescriptorType: %s" EOL
+						"\tEndpointAddress: %s0x%x" EOL
+						"\tAttributes: %s" EOL "\tMaxPacketSize: %d" EOL
+						"\tInterval: %d" EOL "\tRefresh: %d",
+							(int)count, (int)(idesc->bInterfaceNumber), k,
+							destype(epdesc->bDescriptorType),
+							epdir(epdesc->bEndpointAddress),
+							(int)(epdesc->bEndpointAddress),
+							epatt(epdesc->bmAttributes),
+							epdesc->wMaxPacketSize,
+							(int)(epdesc->bInterval),
+							(int)(epdesc->bRefresh));
+				append(buf, tmp, off, len);
+			}
+		}
+	}
+
+	libusb_free_config_descriptor(config);
+	config = NULL;
+
+	err = libusb_get_string_descriptor_ascii(handle, desc.iManufacturer, man, STRBUFLEN);
+	if (err < 0)
+		sprintf((char *)man, "** err(%d)", err);
+
+	err = libusb_get_string_descriptor_ascii(handle, desc.iProduct, prod, STRBUFLEN);
+	if (err < 0)
+		sprintf((char *)prod, "** err(%d)", err);
+
+	err = libusb_get_string_descriptor_ascii(handle, desc.iSerialNumber, ser, STRBUFLEN);
+	if (err < 0)
+		sprintf((char *)ser, "** err(%d)", err);
+
+	sprintf(tmp, EOL "     dev %d: More Info:" EOL "\tManufacturer: '%s'" EOL
+			"\tProduct: '%s'" EOL "\tSerial '%s'",
+				(int)count, man, prod, ser);
+	append(buf, tmp, off, len);
+
+	libusb_close(handle);
+}
+
+// Function to dump all USB devices
+static void usb_all()
+{
+	libusb_device **list;
+	ssize_t count, i;
+	char *buf;
+	size_t len, off;
+
+	count = libusb_get_device_list(NULL, &list);
+	if (count < 0) {
+		applog(LOG_ERR, "USB all: failed, err %d", (int)count);
+		return;
+	}
+
+	if (count == 0)
+		applog(LOG_WARNING, "USB all: found no devices");
+	else
+	{
+		len = 10000;
+		buf = malloc(len+1);
+
+		sprintf(buf, "USB all: found %d devices", (int)count);
+
+		off = strlen(buf);
+
+		for (i = 0; i < count; i++)
+			usb_full(i, list[i], &buf, &off, &len);
+
+		applog(LOG_WARNING, "%s", buf);
+
+		free(buf);
+	}
+
+	libusb_free_device_list(list, 1);
+}
+
+static void cgusb_check_init()
+{
+	mutex_lock(&cgusb_lock);
+
+	if (list_lock == NULL) {
+		list_lock = calloc(1, sizeof(*list_lock));
+		mutex_init(list_lock);
+
+		if (opt_usbdump >= 0) {
+			libusb_set_debug(NULL, opt_usbdump);
+			usb_all();
+		}
+
+		usb_commands = malloc(sizeof(*usb_commands) * C_MAX);
+
+		// use constants so the stat generation is very quick
+		// and the association between number and name can't
+		// be missalined easily
+		usb_commands[C_PING] = C_PING_S;
+		usb_commands[C_CLEAR] = C_CLEAR_S;
+		usb_commands[C_REQUESTVERSION] = C_REQUESTVERSION_S;
+		usb_commands[C_GETVERSION] = C_GETVERSION_S;
+		usb_commands[C_REQUESTFPGACOUNT] = C_REQUESTFPGACOUNT_S;
+		usb_commands[C_GETFPGACOUNT] = C_GETFPGACOUNT_S;
+		usb_commands[C_STARTPROGRAM] = C_STARTPROGRAM_S;
+		usb_commands[C_STARTPROGRAMSTATUS] = C_STARTPROGRAMSTATUS_S;
+		usb_commands[C_PROGRAM] = C_PROGRAM_S;
+		usb_commands[C_PROGRAMSTATUS] = C_PROGRAMSTATUS_S;
+		usb_commands[C_PROGRAMSTATUS2] = C_PROGRAMSTATUS2_S;
+		usb_commands[C_FINALPROGRAMSTATUS] = C_FINALPROGRAMSTATUS_S;
+		usb_commands[C_SETCLOCK] = C_SETCLOCK_S;
+		usb_commands[C_REPLYSETCLOCK] = C_REPLYSETCLOCK_S;
+		usb_commands[C_REQUESTUSERCODE] = C_REQUESTUSERCODE_S;
+		usb_commands[C_GETUSERCODE] = C_GETUSERCODE_S;
+		usb_commands[C_REQUESTTEMPERATURE] = C_REQUESTTEMPERATURE_S;
+		usb_commands[C_GETTEMPERATURE] = C_GETTEMPERATURE_S;
+		usb_commands[C_SENDWORK] = C_SENDWORK_S;
+		usb_commands[C_SENDWORKSTATUS] = C_SENDWORKSTATUS_S;
+		usb_commands[C_REQUESTWORKSTATUS] = C_REQUESTWORKSTATUS_S;
+		usb_commands[C_GETWORKSTATUS] = C_GETWORKSTATUS_S;
+	}
+
+	mutex_unlock(&cgusb_lock);
+}
+
+static bool in_use(libusb_device *dev, bool lock)
+{
+	struct usb_list *usb_tmp;
+	bool used = false;
+	uint8_t bus_number;
+	uint8_t device_address;
+
+	bus_number = libusb_get_bus_number(dev);
+	device_address = libusb_get_device_address(dev);
+
+	if (lock)
+		mutex_lock(list_lock);
+
+	if ((usb_tmp = usb_head))
+		do {
+			if (bus_number == usb_tmp->bus_number
+			&&  device_address == usb_tmp->device_address) {
+				used = true;
+				break;
+			}
+
+			usb_tmp = usb_tmp->next;
+
+		} while (usb_tmp != usb_head);
+
+	if (lock)
+		mutex_unlock(list_lock);
+
+	return used;
+}
+
+static void add_used(libusb_device *dev, bool lock)
+{
+	struct usb_list *usb_tmp;
+	char buf[128];
+	uint8_t bus_number;
+	uint8_t device_address;
+
+	bus_number = libusb_get_bus_number(dev);
+	device_address = libusb_get_device_address(dev);
+
+	if (lock)
+		mutex_lock(list_lock);
+
+	if (in_use(dev, false)) {
+		if (lock)
+			mutex_unlock(list_lock);
+
+		sprintf(buf, "add_used() duplicate bus_number %d device_address %d",
+				bus_number, device_address);
+		quit(1, buf);
+	}
+
+	usb_tmp = malloc(sizeof(*usb_tmp));
+
+	usb_tmp->bus_number = bus_number;
+	usb_tmp->device_address = device_address;
+
+	if (usb_head) {
+		// add to end
+		usb_tmp->prev = usb_head->prev;
+		usb_tmp->next = usb_head;
+		usb_head->prev = usb_tmp;
+		usb_tmp->prev->next = usb_tmp;
+	} else {
+		usb_tmp->prev = usb_tmp;
+		usb_tmp->next = usb_tmp;
+		usb_head = usb_tmp;
+	}
+
+	if (lock)
+		mutex_unlock(list_lock);
+}
+
+static void release(uint8_t bus_number, uint8_t device_address, bool lock)
+{
+	struct usb_list *usb_tmp;
+	bool found = false;
+	char buf[128];
+
+	if (lock)
+		mutex_lock(list_lock);
+
+	usb_tmp = usb_head;
+	if (usb_tmp)
+		do {
+			if (bus_number == usb_tmp->bus_number
+			&&  device_address == usb_tmp->device_address) {
+				found = true;
+				break;
+			}
+
+			usb_tmp = usb_tmp->next;
+
+		} while (usb_tmp != usb_head);
+
+	if (!found) {
+		if (lock)
+			mutex_unlock(list_lock);
+
+		sprintf(buf, "release() unknown: bus_number %d device_address %d",
+				bus_number, device_address);
+		quit(1, buf);
+	}
+
+	if (usb_tmp->next == usb_tmp) {
+		usb_head = NULL;
+	} else {
+		usb_tmp->next->prev = usb_tmp->prev;
+		usb_tmp->prev->next = usb_tmp->next;
+	}
+
+	if (lock)
+		mutex_unlock(list_lock);
+
+	free(usb_tmp);
+}
+
+static void release_dev(libusb_device *dev, bool lock)
+{
+	uint8_t bus_number;
+	uint8_t device_address;
+
+	bus_number = libusb_get_bus_number(dev);
+	device_address = libusb_get_device_address(dev);
+
+	release(bus_number, device_address, lock);
+}
+
+#if 0
+static void release_cgusb(struct cg_usb_device *cgusb, bool lock)
+{
+	release(cgusb->bus_number, cgusb->device_address, lock);
+}
+#endif
+
+static struct cg_usb_device *free_cgusb(struct cg_usb_device *cgusb)
+{
+	if (cgusb->serial_string && cgusb->serial_string != BLANK)
+		free(cgusb->serial_string);
+
+	if (cgusb->manuf_string && cgusb->manuf_string != BLANK)
+		free(cgusb->manuf_string);
+
+	if (cgusb->prod_string && cgusb->prod_string != BLANK)
+		free(cgusb->prod_string);
+
+	free(cgusb->descriptor);
+
+	free(cgusb);
+
+	return NULL;
+}
+
+void usb_uninit(struct cgpu_info *cgpu)
+{
+	libusb_release_interface(cgpu->usbdev->handle, cgpu->usbdev->found->interface);
+	libusb_close(cgpu->usbdev->handle);
+	cgpu->usbdev = free_cgusb(cgpu->usbdev);
+}
+
+bool usb_init(struct cgpu_info *cgpu, struct libusb_device *dev, struct usb_find_devices *found)
+{
+	struct cg_usb_device *cgusb = NULL;
+	struct libusb_config_descriptor *config = NULL;
+	const struct libusb_interface_descriptor *idesc;
+	const struct libusb_endpoint_descriptor *epdesc;
+	unsigned char strbuf[STRBUFLEN+1];
+	int err, i, j, k;
+
+	cgusb = calloc(1, sizeof(*cgusb));
+	cgusb->found = found;
+	cgusb->descriptor = calloc(1, sizeof(*(cgusb->descriptor)));
+
+	err = libusb_get_device_descriptor(dev, cgusb->descriptor);
+	if (err) {
+		applog(LOG_ERR, "USB init failed to get descriptor, err %d", err);
+		goto dame;
+	}
+
+	err = libusb_open(dev, &(cgusb->handle));
+	if (err) {
+		switch (err) {
+			case LIBUSB_ERROR_ACCESS:
+				applog(LOG_ERR, "USB init open device failed, err %d, you dont have priviledge to access the device", err);
+				break;
+#ifdef WIN32
+			// Windows specific message
+			case LIBUSB_ERROR_NOT_SUPPORTED:
+				applog(LOG_ERR, "USB init, open device failed, err %d, you need to install a Windows USB driver for the device", err);
+				break;
+#endif
+			default:
+				applog(LOG_ERR, "USB init, open device failed, err %d", err);
+		}
+
+		goto dame;
+	}
+
+	if (libusb_kernel_driver_active(cgusb->handle, 0) == 1) {
+		applog(LOG_WARNING, "USB init, kernel attached ...");
+		if (libusb_detach_kernel_driver(cgusb->handle, 0) == 0)
+			applog(LOG_WARNING, "USB init, kernel detached successfully");
+		else
+			applog(LOG_WARNING, "USB init, kernel detach failed :(");
+	}
+
+	err = libusb_set_configuration(cgusb->handle, found->config);
+	if (err) {
+		applog(LOG_DEBUG, "USB init, failed to set config to %d, err %d",
+			found->config, err);
+		goto cldame;
+	}
+
+	err = libusb_get_active_config_descriptor(dev, &config);
+	if (err) {
+		applog(LOG_DEBUG, "USB init, failed to get config descriptor %d, err %d",
+			found->config, err);
+		goto cldame;
+	}
+
+	if ((int)(config->bNumInterfaces) < found->interface)
+		goto cldame;
+
+	for (i = 0; i < found->epcount; i++)
+		found->eps[i].found = false;
+
+	for (i = 0; i < config->interface[found->interface].num_altsetting; i++) {
+		idesc = &(config->interface[found->interface].altsetting[i]);
+		for (j = 0; j < (int)(idesc->bNumEndpoints); j++) {
+			epdesc = &(idesc->endpoint[j]);
+			for (k = 0; k < found->epcount; k++) {
+				if (!found->eps[k].found) {
+					if (epdesc->bmAttributes == found->eps[k].att
+					&&  epdesc->wMaxPacketSize >= found->eps[k].size
+					&&  epdesc->bEndpointAddress == found->eps[k].ep) {
+						found->eps[k].found = true;
+						break;
+					}
+				}
+			}
+		}
+	}
+
+	for (i = 0; i < found->epcount; i++)
+		if (found->eps[i].found == false)
+			goto cldame;
+
+	err = libusb_claim_interface(cgusb->handle, found->interface);
+	if (err) {
+		applog(LOG_DEBUG, "USB init, claim interface %d failed, err %d",
+			found->interface, err);
+		goto cldame;
+	}
+
+	cgusb->bus_number = libusb_get_bus_number(dev);
+	cgusb->device_address = libusb_get_device_address(dev);
+	cgusb->usbver = cgusb->descriptor->bcdUSB;
+
+// TODO: allow this with the right version of the libusb include and running library
+//	cgusb->speed = libusb_get_device_speed(dev);
+
+	err = libusb_get_string_descriptor_ascii(cgusb->handle,
+				cgusb->descriptor->iProduct, strbuf, STRBUFLEN);
+	if (err > 0)
+		cgusb->prod_string = strdup((char *)strbuf);
+	else
+		cgusb->prod_string = (char *)BLANK;
+
+	err = libusb_get_string_descriptor_ascii(cgusb->handle,
+				cgusb->descriptor->iManufacturer, strbuf, STRBUFLEN);
+	if (err > 0)
+		cgusb->manuf_string = strdup((char *)strbuf);
+	else
+		cgusb->manuf_string = (char *)BLANK;
+
+	err = libusb_get_string_descriptor_ascii(cgusb->handle,
+				cgusb->descriptor->iSerialNumber, strbuf, STRBUFLEN);
+	if (err > 0)
+		cgusb->serial_string = strdup((char *)strbuf);
+	else
+		cgusb->serial_string = (char *)BLANK;
+
+// TODO: ?
+//	cgusb->fwVersion <- for temp1/temp2 decision? or serial? (driver-modminer.c)
+//	cgusb->interfaceVersion
+
+	applog(LOG_DEBUG, "USB init device bus_number=%d device_address=%d usbver=%04x prod='%s' manuf='%s' serial='%s'", (int)(cgusb->bus_number), (int)(cgusb->device_address), cgusb->usbver, cgusb->prod_string, cgusb->manuf_string, cgusb->serial_string);
+
+	cgpu->usbdev = cgusb;
+
+	libusb_free_config_descriptor(config);
+
+	return true;
+
+cldame:
+
+	libusb_close(cgusb->handle);
+
+dame:
+
+	if (config)
+		libusb_free_config_descriptor(config);
+
+	cgusb = free_cgusb(cgusb);
+
+	return false;
+}
+
+static bool usb_check_device(struct device_api *api, struct libusb_device *dev, struct usb_find_devices *look)
+{
+	struct libusb_device_descriptor desc;
+	int err;
+
+	err = libusb_get_device_descriptor(dev, &desc);
+	if (err) {
+		applog(LOG_DEBUG, "USB check device: Failed to get descriptor, err %d", err);
+		return false;
+	}
+
+	if (desc.idVendor != look->idVendor || desc.idProduct != look->idProduct) {
+		applog(LOG_DEBUG, "%s looking for %04x:%04x but found %04x:%04x instead",
+			api->name, look->idVendor, look->idProduct, desc.idVendor, desc.idProduct);
+
+		return false;
+	}
+
+	applog(LOG_DEBUG, "%s looking for and found %04x:%04x",
+		api->name, look->idVendor, look->idProduct);
+
+	return true;
+}
+
+static struct usb_find_devices *usb_check_each(int drv, struct device_api *api, struct libusb_device *dev)
+{
+	struct usb_find_devices *found;
+	int i;
+
+	for (i = 0; find_dev[i].drv != DRV_LAST; i++)
+		if (find_dev[i].drv == drv) {
+			if (usb_check_device(api, dev, &(find_dev[i]))) {
+				found = malloc(sizeof(*found));
+				memcpy(found, &(find_dev[i]), sizeof(*found));
+				return found;
+			}
+		}
+
+	return NULL;
+}
+
+static struct usb_find_devices *usb_check(__maybe_unused struct device_api *api, __maybe_unused struct libusb_device *dev)
+{
+#ifdef USE_BITFORCE
+	if (api == &bitforce_api)
+		return usb_check_each(DRV_BITFORCE, api, dev);
+#endif
+
+#ifdef USE_ICARUS
+	if (api == &icarus_api)
+		return usb_check_each(DRV_ICARUS, api, dev);
+#endif
+
+#ifdef USE_MODMINER
+	if (api == &modminer_api)
+		return usb_check_each(DRV_MODMINER, api, dev);
+#endif
+
+	return NULL;
+}
+
+void usb_detect(struct device_api *api, bool (*device_detect)(struct libusb_device *, struct usb_find_devices *))
+{
+	libusb_device **list;
+	ssize_t count, i;
+	struct usb_find_devices *found;
+
+	cgusb_check_init();
+
+	count = libusb_get_device_list(NULL, &list);
+	if (count < 0) {
+		applog(LOG_DEBUG, "USB scan devices: failed, err %d", count);
+		return;
+	}
+
+	if (count == 0)
+		applog(LOG_DEBUG, "USB scan devices: found no devices");
+
+	for (i = 0; i < count; i++) {
+		mutex_lock(list_lock);
+
+		if (in_use(list[i], false))
+			mutex_unlock(list_lock);
+		else {
+			add_used(list[i], false);
+
+			mutex_unlock(list_lock);
+
+			found = usb_check(api, list[i]);
+			if (!found)
+				release_dev(list[i], true);
+			else
+				if (!device_detect(list[i], found))
+					release_dev(list[i], true);
+		}
+	}
+
+	libusb_free_device_list(list, 1);
+}
+
+// Set this to 0 to remove stats processing
+#define DO_USB_STATS 1
+
+#if DO_USB_STATS
+#define USB_STATS(sgpu, sta, fin, err, cmd, seq) stats(cgpu, sta, fin, err, cmd, seq)
+#define STATS_TIMEVAL(tv) gettimeofday(tv, NULL)
+#else
+#define USB_STATS(sgpu, sta, fin, err, cmd, seq)
+#define STATS_TIMEVAL(tv)
+#endif
+
+// The stat data can be spurious due to not locking it before copying it -
+// however that would require the stat() function to also lock and release
+// a mutex every time a usb read or write is called which would slow
+// things down more
+struct api_data *api_usb_stats(__maybe_unused int *count)
+{
+#if DO_USB_STATS
+	struct cg_usb_stats_details *details;
+	struct cg_usb_stats *sta;
+	struct api_data *root = NULL;
+	int device;
+	int cmdseq;
+
+	cgusb_check_init();
+
+	if (next_stat == 0)
+		return NULL;
+
+	while (*count < next_stat * C_MAX * 2) {
+		device = *count / (C_MAX * 2);
+		cmdseq = *count % (C_MAX * 2);
+
+		(*count)++;
+
+		sta = &(usb_stats[device]);
+		details = &(sta->details[cmdseq]);
+
+		// Only show stats that have results
+		if (details->item[CMD_CMD].count == 0 &&
+		    details->item[CMD_TIMEOUT].count == 0 &&
+		    details->item[CMD_ERROR].count == 0)
+			continue;
+
+		root = api_add_string(root, "Name", sta->name, false);
+		root = api_add_int(root, "ID", &(sta->device_id), false);
+		root = api_add_const(root, "Stat", usb_commands[cmdseq/2], false);
+		root = api_add_int(root, "Seq", &(details->seq), true);
+		root = api_add_uint64(root, "Count",
+					&(details->item[CMD_CMD].count), true);
+		root = api_add_double(root, "Total Delay",
+					&(details->item[CMD_CMD].total_delay), true);
+		root = api_add_double(root, "Min Delay",
+					&(details->item[CMD_CMD].min_delay), true);
+		root = api_add_double(root, "Max Delay",
+					&(details->item[CMD_CMD].max_delay), true);
+		root = api_add_uint64(root, "Timeout Count",
+					&(details->item[CMD_TIMEOUT].count), true);
+		root = api_add_double(root, "Timeout Total Delay",
+					&(details->item[CMD_TIMEOUT].total_delay), true);
+		root = api_add_double(root, "Timeout Min Delay",
+					&(details->item[CMD_TIMEOUT].min_delay), true);
+		root = api_add_double(root, "Timeout Max Delay",
+					&(details->item[CMD_TIMEOUT].max_delay), true);
+		root = api_add_uint64(root, "Error Count",
+					&(details->item[CMD_ERROR].count), true);
+		root = api_add_double(root, "Error Total Delay",
+					&(details->item[CMD_ERROR].total_delay), true);
+		root = api_add_double(root, "Error Min Delay",
+					&(details->item[CMD_ERROR].min_delay), true);
+		root = api_add_double(root, "Error Max Delay",
+					&(details->item[CMD_ERROR].max_delay), true);
+		root = api_add_timeval(root, "First Command",
+					&(details->item[CMD_CMD].first), true);
+		root = api_add_timeval(root, "Last Command",
+					&(details->item[CMD_CMD].last), true);
+		root = api_add_timeval(root, "First Timeout",
+					&(details->item[CMD_TIMEOUT].first), true);
+		root = api_add_timeval(root, "Last Timeout",
+					&(details->item[CMD_TIMEOUT].last), true);
+		root = api_add_timeval(root, "First Error",
+					&(details->item[CMD_ERROR].first), true);
+		root = api_add_timeval(root, "Last Error",
+					&(details->item[CMD_ERROR].last), true);
+
+		return root;
+	}
+#endif
+	return NULL;
+}
+
+#if DO_USB_STATS
+static void newstats(struct cgpu_info *cgpu)
+{
+	int i;
+
+	cgpu->usbstat = ++next_stat;
+
+	usb_stats = realloc(usb_stats, sizeof(*usb_stats) * next_stat);
+	usb_stats[next_stat-1].name = cgpu->api->name;
+	usb_stats[next_stat-1].device_id = -1;
+	usb_stats[next_stat-1].details = calloc(1, sizeof(struct cg_usb_stats_details) * C_MAX * 2);
+	for (i = 1; i < C_MAX * 2; i += 2)
+		usb_stats[next_stat-1].details[i].seq = 1;
+}
+#endif
+
+void update_usb_stats(__maybe_unused struct cgpu_info *cgpu)
+{
+#if DO_USB_STATS
+	// we don't know the device_id until after add_cgpu()
+	usb_stats[cgpu->usbstat - 1].device_id = cgpu->device_id;
+#endif
+}
+
+#if DO_USB_STATS
+static void stats(struct cgpu_info *cgpu, struct timeval *tv_start, struct timeval *tv_finish, int err, enum usb_cmds cmd, int seq)
+{
+	struct cg_usb_stats_details *details;
+	double diff;
+	int item;
+
+	if (cgpu->usbstat < 1)
+		newstats(cgpu);
+
+	details = &(usb_stats[cgpu->usbstat - 1].details[cmd * 2 + seq]);
+
+	diff = tdiff(tv_finish, tv_start);
+
+	switch (err) {
+		case LIBUSB_SUCCESS:
+			item = CMD_CMD;
+			break;
+		case LIBUSB_ERROR_TIMEOUT:
+			item = CMD_TIMEOUT;
+			break;
+		default:
+			item = CMD_ERROR;
+			break;
+	}
+
+	if (details->item[item].count == 0) {
+		details->item[item].min_delay = diff;
+		memcpy(&(details->item[item].first), tv_start, sizeof(*tv_start));
+	} else if (diff < details->item[item].min_delay)
+		details->item[item].min_delay = diff;
+
+	if (diff > details->item[item].max_delay)
+		details->item[item].max_delay = diff;
+
+	details->item[item].total_delay += diff;
+	memcpy(&(details->item[item].last), tv_start, sizeof(tv_start));
+	details->item[item].count++;
+}
+#endif
+
+int _usb_read(struct cgpu_info *cgpu, int ep, char *buf, size_t bufsiz, int *processed, unsigned int timeout, int eol, enum usb_cmds cmd)
+{
+	struct cg_usb_device *usbdev = cgpu->usbdev;
+#if DO_USB_STATS
+	struct timeval tv_start, tv_finish;
+#endif
+	int err, got, tot;
+	bool first = true;
+
+	if (eol == -1) {
+		got = 0;
+		STATS_TIMEVAL(&tv_start);
+		err = libusb_bulk_transfer(usbdev->handle,
+				usbdev->found->eps[ep].ep,
+				(unsigned char *)buf,
+				bufsiz, &got,
+				timeout == DEVTIMEOUT ? usbdev->found->timeout : timeout);
+		STATS_TIMEVAL(&tv_finish);
+		USB_STATS(cgpu, &tv_start, &tv_finish, err, cmd, SEQ0);
+
+		*processed = got;
+
+		return err;
+	}
+
+	tot = 0;
+	while (bufsiz) {
+		got = 0;
+		STATS_TIMEVAL(&tv_start);
+		err = libusb_bulk_transfer(usbdev->handle,
+				usbdev->found->eps[ep].ep,
+				(unsigned char *)buf,
+				1, &got,
+				timeout == DEVTIMEOUT ? usbdev->found->timeout : timeout);
+		STATS_TIMEVAL(&tv_finish);
+		USB_STATS(cgpu, &tv_start, &tv_finish, err, cmd, first ? SEQ0 : SEQ1);
+
+		tot += got;
+
+		if (err)
+			break;
+
+		if (eol == buf[0])
+			break;
+
+		buf += got;
+		bufsiz -= got;
+
+		first = false;
+	}
+
+	*processed = tot;
+
+	return err;
+}
+
+int _usb_write(struct cgpu_info *cgpu, int ep, char *buf, size_t bufsiz, int *processed, unsigned int timeout, enum usb_cmds cmd)
+{
+	struct cg_usb_device *usbdev = cgpu->usbdev;
+#if DO_USB_STATS
+	struct timeval tv_start, tv_finish;
+#endif
+	int err, sent;
+
+	sent = 0;
+	STATS_TIMEVAL(&tv_start);
+	err = libusb_bulk_transfer(usbdev->handle,
+			usbdev->found->eps[ep].ep,
+			(unsigned char *)buf,
+			bufsiz, &sent,
+			timeout == DEVTIMEOUT ? usbdev->found->timeout : timeout);
+	STATS_TIMEVAL(&tv_finish);
+	USB_STATS(cgpu, &tv_start, &tv_finish, err, cmd, SEQ0);
+
+	*processed = sent;
+
+	return err;
+}
+
+void usb_cleanup()
+{
+	// TODO:
+}
diff --git a/usbutils.h b/usbutils.h
new file mode 100644
index 0000000..71fd969
--- /dev/null
+++ b/usbutils.h
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2012 Andrew Smith
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option)
+ * any later version.  See COPYING for more details.
+ */
+
+#ifndef USBUTILS_H
+#define USBUTILS_H
+
+#include <libusb.h>
+
+// Use the device defined timeout
+#define DEVTIMEOUT 0
+
+// For endpoints defined in usb_find_devices.eps,
+// the first two must be the default IN and OUT
+#define DEFAULT_EP_IN 0
+#define DEFAULT_EP_OUT 1
+
+struct usb_endpoints {
+	uint8_t att;
+	uint16_t size;
+	unsigned char ep;
+	bool found;
+};
+
+struct usb_find_devices {
+	int drv;
+	const char *name;
+	uint16_t idVendor;
+	uint16_t idProduct;
+	int config;
+	int interface;
+	unsigned int timeout;
+	int epcount;
+	struct usb_endpoints *eps;
+};
+
+struct cg_usb_device {
+	struct usb_find_devices *found;
+	libusb_device_handle *handle;
+	pthread_mutex_t *mutex;
+	struct libusb_device_descriptor *descriptor;
+	uint8_t bus_number;
+	uint8_t device_address;
+	uint16_t usbver;
+	int speed;
+	char *prod_string;
+	char *manuf_string;
+	char *serial_string;
+	unsigned char fwVersion;	// ??
+	unsigned char interfaceVersion;	// ??
+};
+
+enum usb_cmds {
+	C_PING = 0,
+	C_CLEAR,
+	C_REQUESTVERSION,
+	C_GETVERSION,
+	C_REQUESTFPGACOUNT,
+	C_GETFPGACOUNT,
+	C_STARTPROGRAM,
+	C_STARTPROGRAMSTATUS,
+	C_PROGRAM,
+	C_PROGRAMSTATUS,
+	C_PROGRAMSTATUS2,
+	C_FINALPROGRAMSTATUS,
+	C_SETCLOCK,
+	C_REPLYSETCLOCK,
+	C_REQUESTUSERCODE,
+	C_GETUSERCODE,
+	C_REQUESTTEMPERATURE,
+	C_GETTEMPERATURE,
+	C_SENDWORK,
+	C_SENDWORKSTATUS,
+	C_REQUESTWORKSTATUS,
+	C_GETWORKSTATUS,
+	C_MAX
+};
+
+struct device_api;
+struct cgpu_info;
+
+void usb_uninit(struct cgpu_info *cgpu);
+bool usb_init(struct cgpu_info *cgpu, struct libusb_device *dev, struct usb_find_devices *found);
+void usb_detect(struct device_api *api, bool (*device_detect)(struct libusb_device *, struct usb_find_devices *));
+struct api_data *api_usb_stats(int *count);
+void update_usb_stats(struct cgpu_info *cgpu);
+int _usb_read(struct cgpu_info *cgpu, int ep, char *buf, size_t bufsiz, int *processed, unsigned int timeout, int eol, enum usb_cmds);
+int _usb_write(struct cgpu_info *cgpu, int ep, char *buf, size_t bufsiz, int *processed, unsigned int timeout, enum usb_cmds);
+void usb_cleanup();
+
+#define usb_read(cgpu, buf, bufsiz, read, cmd) \
+	_usb_read(cgpu, DEFAULT_EP_IN, buf, bufsiz, read, DEVTIMEOUT, -1, cmd)
+
+#define usb_read_ep(cgpu, ep, buf, bufsiz, read, cmd) \
+	_usb_read(cgpu, ep, buf, bufsiz, read, DEVTIMEOUT, -1, cmd)
+
+#define usb_read_timeout(cgpu, buf, bufsiz, read, timeout, cmd) \
+	_usb_read(cgpu, DEFAULT_EP_IN, buf, bufsiz, read, timeout, -1, cmd)
+
+#define usb_read_ep_timeout(cgpu, ep, buf, bufsiz, read, timeout, cmd) \
+	_usb_read(cgpu, ep, buf, bufsiz, read, timeout, -1, cmd)
+
+#define usb_write(cgpu, buf, bufsiz, wrote, cmd) \
+	_usb_write(cgpu, DEFAULT_EP_OUT, buf, bufsiz, wrote, DEVTIMEOUT, cmd)
+
+#define usb_write_ep(cgpu, ep, buf, bufsiz, wrote, cmd) \
+	_usb_write(cgpu, ep, buf, bufsiz, wrote, DEVTIMEOUT, cmd)
+
+#define usb_write_timeout(cgpu, buf, bufsiz, wrote, timeout, cmd) \
+	_usb_write(cgpu, DEFAULT_EP_OUT, buf, bufsiz, wrote, timeout, cmd)
+
+#define usb_write_ep_timeout(cgpu, ep, buf, bufsiz, wrote, timeout, cmd) \
+	_usb_write(cgpu, ep, buf, bufsiz, wrote, timeout, cmd)
+
+#endif