Commit 324d849bf2470279e92d70cba594d45089645c5a
2011-08-20T23:03:49
Attempt to restart a GPU once every minute while it's sick. Don't kill off the reinit thread if it fails to init a GPU but returns safely. Only declare a GPU dead if there's been no sign of activity from the reinit thread for 10 mins.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
diff --git a/main.c b/main.c
index 7177255..842c53c 100644
--- a/main.c
+++ b/main.c
@@ -3976,6 +3976,8 @@ select_cgpu:
thr = &thr_info[thr_id];
thr->rolling = thr->cgpu->rolling = 0;
+ /* Reports the last time we tried to revive a sick GPU */
+ gettimeofday(&thr->sick, NULL);
if (!pthread_cancel(*thr->pth)) {
applog(LOG_WARNING, "Thread %d still exists, killing it off", thr_id);
} else
@@ -4004,7 +4006,7 @@ select_cgpu:
clStates[thr_id] = initCl(gpu, name, sizeof(name));
if (!clStates[thr_id]) {
applog(LOG_ERR, "Failed to reinit GPU thread %d", thr_id);
- goto out;
+ goto select_cgpu;
}
applog(LOG_INFO, "initCl() finished. Found %s", name);
@@ -4139,14 +4141,15 @@ static void *watchdog_thread(void *userdata)
thr->rolling = thr->cgpu->rolling = 0;
gpus[gpu].status = LIFE_SICK;
applog(LOG_ERR, "Thread %d idle for more than 60 seconds, GPU %d declared SICK!", i, gpu);
- /* Sent it a ping, it might respond */
- tq_push(thr->q, &ping);
- } else if (now.tv_sec - thr->last.tv_sec > 300 && gpus[i].status == LIFE_SICK) {
- gpus[gpu].status = LIFE_DEAD;
- applog(LOG_ERR, "Thread %d idle for more than 5 minutes, GPU %d declared DEAD!", i, gpu);
applog(LOG_ERR, "Attempting to restart GPU");
+ gettimeofday(&thr->sick, NULL);
+ reinit_device(thr->cgpu);
+ } else if (now.tv_sec - thr->sick.tv_sec > 600 && gpus[i].status == LIFE_SICK) {
+ gpus[gpu].status = LIFE_DEAD;
+ applog(LOG_ERR, "Thread %d not responding for more than 10 minutes, GPU %d declared DEAD!", i, gpu);
+ } else if (now.tv_sec - thr->sick.tv_sec > 60 && gpus[i].status == LIFE_SICK) {
+ /* Attempt to restart a GPU once every minute */
reinit_device(thr->cgpu);
- break;
}
}
}
diff --git a/miner.h b/miner.h
index d96ad7f..49bce03 100644
--- a/miner.h
+++ b/miner.h
@@ -162,6 +162,8 @@ struct thr_info {
struct thread_q *q;
struct cgpu_info *cgpu;
struct timeval last;
+ struct timeval sick;
+
bool getwork;
double rolling;
};