Commit 59b640870c1703097a9f48c1ed0a96aa548e8675
2011-07-30T13:28:25
So we can pthread cancel a stuck thread, but can't pthread join it. And we can't tq freeze the old queue or we could hang here. And we need to re-initialise all the opencl contexts so send a test command and if it succeeds reinit the card.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
diff --git a/main.c b/main.c
index 0605457..03db486 100644
--- a/main.c
+++ b/main.c
@@ -3133,13 +3133,13 @@ static void *gpuminer_thread(void *userdata)
}
}
if (unlikely(!gpu_devices[gpu])) {
- applog(LOG_WARNING, "Thread %d being disabled\n", thr_id);
+ applog(LOG_WARNING, "Thread %d being disabled", thr_id);
mythr->rolling = mythr->cgpu->rolling = 0;
if (opt_debug)
applog(LOG_DEBUG, "Popping wakeup ping in gpuminer thread");
tq_pop(mythr->q, NULL); /* Ignore ping that's popped */
- applog(LOG_WARNING, "Thread %d being re-enabled\n", thr_id);
+ applog(LOG_WARNING, "Thread %d being re-enabled", thr_id);
}
}
out:
@@ -3361,7 +3361,9 @@ static void *reinit_gpu(void *userdata)
int thr_id;
_clState *clState;
- gpus[gpu].status = LIFE_DEAD;
+ /* Send threads message to stop */
+ gpu_devices[gpu] = false;
+ sleep(5);
for (thr_id = 0; thr_id < gpu_threads; thr_id ++) {
if (dev_from_id(thr_id) != gpu)
@@ -3370,11 +3372,14 @@ static void *reinit_gpu(void *userdata)
clState = clStates[thr_id];
thr = &thr_info[thr_id];
thr->rolling = thr->cgpu->rolling = 0;
- tq_freeze(thr->q);
if (!pthread_cancel(*thr->pth)) {
- pthread_join(*thr->pth, NULL);
+ applog(LOG_WARNING, "Thread still exists, killing it off");
free(thr->q);
- }
+ } else
+ applog(LOG_WARNING, "Thread no longer exists");
+ sleep(5);
+ /* Lose this ram cause we may get stuck here! */
+ //tq_freeze(thr->q);
thr->q = tq_new();
if (!thr->q)
@@ -3408,6 +3413,14 @@ static void *reinit_gpu(void *userdata)
applog(LOG_WARNING, "Thread %d restarted", thr_id);
}
+ /* Try to re-enable it */
+ gpu_devices[gpu] = true;
+ for (thr_id = 0; thr_id < gpu_threads; thr_id ++) {
+ thr = &thr_info[thr_id];
+ if (dev_from_id(thr_id) == gpu)
+ tq_push(thr->q, &ping);
+ }
+
return NULL;
}
#endif
@@ -3523,9 +3536,9 @@ static void *watchdog_thread(void *userdata)
applog(LOG_ERR, "Thread %d idle for more than 60 seconds, GPU %d declared SICK!", i, gpu);
/* Sent it a ping, it might respond */
tq_push(thr->q, &ping);
- } else if (now.tv_sec - thr->last.tv_sec > 600 && gpus[i].status == LIFE_SICK) {
+ } else if (now.tv_sec - thr->last.tv_sec > 300 && gpus[i].status == LIFE_SICK) {
gpus[gpu].status = LIFE_DEAD;
- applog(LOG_ERR, "Thread %d idle for more than 10 minutes, GPU %d declared DEAD!", i, gpu);
+ applog(LOG_ERR, "Thread %d idle for more than 5 minutes, GPU %d declared DEAD!", i, gpu);
applog(LOG_ERR, "Attempting to restart GPU");
reinit_device(thr->cgpu);
break;