Commit a466942fd880d157cc0b5968805b2159f556fc20
2011-08-12T20:20:42
When pinging a sick cpu, flush finish and then ping it in a separate thread in the hope it recovers, but without blocking code elsewhere.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
diff --git a/main.c b/main.c
index 5d081aa..7058fcf 100644
--- a/main.c
+++ b/main.c
@@ -3415,10 +3415,46 @@ static void *reinit_gpu(void *userdata)
return NULL;
}
+
+static void *ping_gputhread(void *userdata)
+{
+ struct cgpu_info *cgpu = (struct cgpu_info *)userdata;
+ int gpu = cgpu->cpu_gpu;
+ struct thr_info *thr;
+ _clState *clState;
+ int thr_id;
+
+ for (thr_id = 0; thr_id < gpu_threads; thr_id ++) {
+ if (dev_from_id(thr_id) != gpu)
+ continue;
+
+ thr = &thr_info[thr_id];
+ clState = clStates[thr_id];
+ tq_push(thr->q, &ping);
+ applog(LOG_WARNING, "Attempting to flush command queue of thread %d", thr_id);
+ clFlush(clState->commandQueue);
+ clFinish(clState->commandQueue);
+ tq_push(thr->q, &ping);
+ }
+
+ return NULL;
+}
+
+static void ping_gpu(struct cgpu_info *cgpu)
+{
+ pthread_t ping_thread;
+
+ if (unlikely(pthread_create(&ping_thread, NULL, ping_gputhread, (void *)cgpu)))
+ applog(LOG_ERR, "Failed to create ping thread");
+}
#else
static void *reinit_gpu(void *userdata)
{
}
+
+static void ping_gpu(struct cgpu_info *cgpu)
+{
+}
#endif
static void reinit_device(struct cgpu_info *cgpu)
@@ -3531,7 +3567,7 @@ static void *watchdog_thread(void *userdata)
gpus[gpu].status = LIFE_SICK;
applog(LOG_ERR, "Thread %d idle for more than 60 seconds, GPU %d declared SICK!", i, gpu);
/* Sent it a ping, it might respond */
- tq_push(thr->q, &ping);
+ ping_gpu(thr->cgpu);
} else if (now.tv_sec - thr->last.tv_sec > 300 && gpus[i].status == LIFE_SICK) {
gpus[gpu].status = LIFE_DEAD;
applog(LOG_ERR, "Thread %d idle for more than 5 minutes, GPU %d declared DEAD!", i, gpu);