From 574aa620f580c23aac53fc0927f34c9cf1808a26 Mon Sep 17 00:00:00 2001 From: Ingo Albrecht Date: Fri, 6 Jun 2014 02:54:01 +0200 Subject: [PATCH 1/6] crt: Never call pthread_detach We do not need to detach because our threads are all started detached. All resources held by them are managed by the GC or by explicit code. --- sources/lib/run-time/posix-threads.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sources/lib/run-time/posix-threads.c b/sources/lib/run-time/posix-threads.c index 2f93c78e8e..826440c637 100644 --- a/sources/lib/run-time/posix-threads.c +++ b/sources/lib/run-time/posix-threads.c @@ -726,12 +726,9 @@ D primitive_thread_join_multiple(D v) /* 4.5 */ void primitive_detach_thread(D t) { - DTHREAD* thread = t; - THREAD* rthread; - assert(thread != NULL); - rthread = (THREAD*)(thread->handle2); + DTHREAD *thread = (DTHREAD *)t; - pthread_detach(rthread->tid); + assert(thread != NULL); } /* 5 */ From 4ada2cda2fc435c209698271f177e362a9f8124d Mon Sep 17 00:00:00 2001 From: Ingo Albrecht Date: Fri, 6 Jun 2014 02:59:17 +0200 Subject: [PATCH 2/6] crt: Clarify how TLV grow synchronization works --- sources/lib/run-time/posix-threads.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/sources/lib/run-time/posix-threads.c b/sources/lib/run-time/posix-threads.c index 826440c637..3e659d9c18 100644 --- a/sources/lib/run-time/posix-threads.c +++ b/sources/lib/run-time/posix-threads.c @@ -319,7 +319,10 @@ static void grow_all_tlv_vectors(size_t newsize) list = list->next; } - // Let writes proceed again + // Let writes proceed again, but only at a moment + // where TLV_GROW is undisturbed. This ensures + // that we are not between an increment and a + // decrement on the TLV writer side. while (atomic_cas(&tlv_writer_counter, 0, TLV_GROW) != TLV_GROW); } @@ -1566,10 +1569,19 @@ D primitive_read_thread_variable(D h) /* 35 */ -static void primitive_write_thread_variable_internal(void) +static void primitive_write_thread_variable_blocked(void) { + // Spin until we see normal writer counts again do { + // This decrement will undo the increment done before + // entering this function or on the previous loop iteration. + // This will disturb the writer count even if we do not + // have a lock, but only if the count is negative. + // Since TLV_GROW is large we will never change the sign. if (atomic_decrement(&tlv_writer_counter) < 0) { + // Synchronize with TLV growing by waiting + // on the TLV list lock whenever we encounter + // a negative writer count. pthread_mutex_lock(&tlv_vector_list_lock); pthread_mutex_unlock(&tlv_vector_list_lock); } @@ -1581,9 +1593,11 @@ D primitive_write_thread_variable(D h, D nv) TLV_VECTOR tlv_vector; uintptr_t offset; - // Wait until there are no other writers + // Acquire write lock by incrementing if (atomic_increment(&tlv_writer_counter) < 0) { - primitive_write_thread_variable_internal(); + // If the count is negative then another + // thread is growing TLVs. Wait for it. + primitive_write_thread_variable_blocked(); } // The variable handle is the byte offset where the variable's value is From 6297b83a5ae38823904b26605ef39a33f118cd30 Mon Sep 17 00:00:00 2001 From: Ingo Albrecht Date: Fri, 6 Jun 2014 03:11:42 +0200 Subject: [PATCH 3/6] crt: Made setting mark bit clearer --- sources/lib/run-time/posix-threads.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/lib/run-time/posix-threads.c b/sources/lib/run-time/posix-threads.c index 3e659d9c18..076b441938 100644 --- a/sources/lib/run-time/posix-threads.c +++ b/sources/lib/run-time/posix-threads.c @@ -715,7 +715,7 @@ D primitive_thread_join_multiple(D v) for (i = 0; i < size; i++) { state = (uintptr_t)threads[i]->handle1; - threads[i]->handle1 = (void *)(state ^ MARKED); + threads[i]->handle1 = (void *)(state & ~MARKED); } if (pthread_mutex_unlock(&thread_join_lock) != 0) { From 73fc487e95fa290f625e81d914845ddb58618373 Mon Sep 17 00:00:00 2001 From: Ingo Albrecht Date: Fri, 6 Jun 2014 03:12:41 +0200 Subject: [PATCH 4/6] crt: Clarified some other bit operations on thread state --- sources/lib/run-time/posix-threads.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sources/lib/run-time/posix-threads.c b/sources/lib/run-time/posix-threads.c index 076b441938..e6de5e10e3 100644 --- a/sources/lib/run-time/posix-threads.c +++ b/sources/lib/run-time/posix-threads.c @@ -630,6 +630,7 @@ D primitive_thread_join_single(D t) return GENERAL_ERROR; } + state = (uintptr_t)thread->handle1; thread->handle1 = (void *)(state | MARKED); completed = state & COMPLETED; while (!completed) { @@ -640,7 +641,8 @@ D primitive_thread_join_single(D t) completed = (uintptr_t)thread->handle1 & COMPLETED; } - thread->handle1 = (void *)((uintptr_t)thread->handle1 ^ (JOINED | MARKED)); + state = (uintptr_t)thread->handle1; + thread->handle1 = (void *)((state | JOINED) & ~MARKED); if (pthread_mutex_unlock(&thread_join_lock) != 0) { MSG0("thread-join-single: error releasing thread join lock\n"); From f931ea209195ae8c749094222c0ab24055a7a643 Mon Sep 17 00:00:00 2001 From: Ingo Albrecht Date: Fri, 6 Jun 2014 03:13:34 +0200 Subject: [PATCH 5/6] crt: Some comments about join synchronization --- sources/lib/run-time/posix-threads.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/sources/lib/run-time/posix-threads.c b/sources/lib/run-time/posix-threads.c index e6de5e10e3..ce1c643a52 100644 --- a/sources/lib/run-time/posix-threads.c +++ b/sources/lib/run-time/posix-threads.c @@ -618,11 +618,13 @@ D primitive_thread_join_single(D t) assert(thread != NULL); + /* Acquire the join lock */ if (pthread_mutex_lock(&thread_join_lock) != 0) { MSG0("thread-join-single: error obtaining thread join lock\n"); return GENERAL_ERROR; } + /* Make sure the thread isn't already part of a join operation */ state = (uintptr_t)thread->handle1; if (state & MARKED || state & JOINED) { pthread_mutex_unlock(&thread_join_lock); @@ -630,8 +632,11 @@ D primitive_thread_join_single(D t) return GENERAL_ERROR; } + /* Mark the thread as joining */ state = (uintptr_t)thread->handle1; thread->handle1 = (void *)(state | MARKED); + + /* Spin until the thread is joined */ completed = state & COMPLETED; while (!completed) { if (pthread_cond_wait(&thread_exit_event, &thread_join_lock)) { @@ -641,9 +646,11 @@ D primitive_thread_join_single(D t) completed = (uintptr_t)thread->handle1 & COMPLETED; } + /* Mark the thread as joined and remove the join mark */ state = (uintptr_t)thread->handle1; thread->handle1 = (void *)((state | JOINED) & ~MARKED); + /* Release the join lock */ if (pthread_mutex_unlock(&thread_join_lock) != 0) { MSG0("thread-join-single: error releasing thread join lock\n"); return GENERAL_ERROR; @@ -668,14 +675,13 @@ D primitive_thread_join_multiple(D v) size = ((uintptr_t)(thread_vector->size)) >> 2; threads = (volatile DTHREAD **)(thread_vector->data); + /* Acquire the join lock */ if (pthread_mutex_lock(&thread_join_lock)) { MSG0("thread-join-multiple: error obtaining thread join lock\n"); return GENERAL_ERROR; } - /* Make sure none of the threads is already - * part of a join operation - */ + /* Ensure none of the threads are already joining or joined */ for (i = 0; i < size; i++) { state = (uintptr_t)threads[i]->handle1; if (state & MARKED || state & JOINED) { @@ -684,13 +690,13 @@ D primitive_thread_join_multiple(D v) } } - /* Now mark the threads as being part of a join - */ + /* Now mark the threads as joining */ for (i = 0; i < size; i++) { state = (uintptr_t)threads[i]->handle1; threads[i]->handle1 = (void *)(state | MARKED); } + /* Spin until a thread is joined */ for (i = 0; i < size; i++) { state = (uintptr_t)threads[i]->handle1; if (state & COMPLETED) { @@ -712,14 +718,17 @@ D primitive_thread_join_multiple(D v) } } + /* Mark the joined thread as such */ state = (uintptr_t)joined_thread->handle1; joined_thread->handle1 = (void *)(state | JOINED); + /* Remove join mark on all the threads so we can retry */ for (i = 0; i < size; i++) { state = (uintptr_t)threads[i]->handle1; threads[i]->handle1 = (void *)(state & ~MARKED); } + /* Release the join lock */ if (pthread_mutex_unlock(&thread_join_lock) != 0) { MSG0("thread-join-multiple: error releasing thread join lock\n"); return GENERAL_ERROR; From 34604d93a4036c758b78dc8d4b8bfea692f20836 Mon Sep 17 00:00:00 2001 From: Ingo Albrecht Date: Fri, 6 Jun 2014 03:18:57 +0200 Subject: [PATCH 6/6] crt: Clarified thread join loops By converting them to do {} while (). --- sources/lib/run-time/posix-threads.c | 43 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/sources/lib/run-time/posix-threads.c b/sources/lib/run-time/posix-threads.c index ce1c643a52..53e88a3581 100644 --- a/sources/lib/run-time/posix-threads.c +++ b/sources/lib/run-time/posix-threads.c @@ -637,14 +637,16 @@ D primitive_thread_join_single(D t) thread->handle1 = (void *)(state | MARKED); /* Spin until the thread is joined */ - completed = state & COMPLETED; - while (!completed) { - if (pthread_cond_wait(&thread_exit_event, &thread_join_lock)) { - MSG0("thread-join-single: error waiting for thread exit event\n"); - return GENERAL_ERROR; + do { + state = (uintptr_t)thread->handle1; + completed = state & COMPLETED; + if(!completed) { + if (pthread_cond_wait(&thread_exit_event, &thread_join_lock)) { + MSG0("thread-join-single: error waiting for thread exit event\n"); + return GENERAL_ERROR; + } } - completed = (uintptr_t)thread->handle1 & COMPLETED; - } + } while (!completed); /* Mark the thread as joined and remove the join mark */ state = (uintptr_t)thread->handle1; @@ -697,26 +699,23 @@ D primitive_thread_join_multiple(D v) } /* Spin until a thread is joined */ - for (i = 0; i < size; i++) { - state = (uintptr_t)threads[i]->handle1; - if (state & COMPLETED) { - joined_thread = threads[i]; - break; - } - } - - while (joined_thread == NULL) { - if (pthread_cond_wait(&thread_exit_event, &thread_join_lock)) { - MSG0("thread-join-multiple: error waiting for thread exit event\n"); - return GENERAL_ERROR; - } + do { + /* Check thread states */ for (i = 0; i < size; i++) { - if ((uintptr_t)threads[i]->handle1 & COMPLETED) { + state = (uintptr_t)threads[i]->handle1; + if (state & COMPLETED) { joined_thread = threads[i]; break; } } - } + /* Wait for join condition unless we already found a thread */ + if(joined_thread == NULL) { + if (pthread_cond_wait(&thread_exit_event, &thread_join_lock)) { + MSG0("thread-join-multiple: error waiting for thread exit event\n"); + return GENERAL_ERROR; + } + } + } while (joined_thread == NULL); /* Mark the joined thread as such */ state = (uintptr_t)joined_thread->handle1;