Sat Jul 25 00:00:13 1998 Ross Johnson <rpj@ixobrychus.canberra.edu.au>

	* create.c (_pthread_start_call): Set thread priority.  Ensure our
 	thread entry is removed from the thread table but only if
 	pthread_detach() was called and there are no waiting joins.
	(pthread_create): Set detach flag in thread entry if the
	thread is created PTHREAD_CREATE_DETACHED.

	* pthread.h (pthread_attr_t): Rename member "detachedstate".

	* attr.c (pthread_attr_init): Rename attr members.

	* exit.c (pthread_exit): Fix indirection mistake.

	* implement.h (_PTHREAD_THREADS_TABLE_INDEX): Add.

	* exit.c (_pthread_vacuum): Fix incorrect args to
	_pthread_handler_pop_all() calls.
	Make thread entry removal conditional.

	* sync.c (pthread_join): Add multiple join and async detach handling.

	* implement.h (_PTHREAD_THREADS_TABLE_INDEX): Add.

	* global.c (_pthread_threads_mutex_table): Add.

	* implement.h (_pthread_once_flag): Remove.
	(_pthread_once_lock): Ditto.
	(_pthread_threads_mutex_table): Add.

	* global.c (_pthread_once_flag): Remove.
	(_pthread_once_lock): Ditto.

	* sync.c (pthread_join): Fix tests involving new return value
	from _pthread_find_thread_entry().
	(pthread_detach): Ditto.

	* private.c (_pthread_find_thread_entry): Failure return code
	changed from -1 to NULL.

9 files changed, 260 insertions, 96 deletions

diff --git a/ChangeLog b/ChangeLog
index 6ddaa40..3f76175 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,43 @@
+Sat Jul 25 00:00:13 1998  Ross Johnson  <rpj@ixobrychus.canberra.edu.au>
+
+	* create.c (_pthread_start_call): Set thread priority.  Ensure our
+ 	thread entry is removed from the thread table but only if
+ 	pthread_detach() was called and there are no waiting joins.
+	(pthread_create): Set detach flag in thread entry if the 
+	thread is created PTHREAD_CREATE_DETACHED.
+
+	* pthread.h (pthread_attr_t): Rename member "detachedstate".
+
+	* attr.c (pthread_attr_init): Rename attr members.
+
+	* exit.c (pthread_exit): Fix indirection mistake.
+
+	* implement.h (_PTHREAD_THREADS_TABLE_INDEX): Add.
+
+	* exit.c (_pthread_vacuum): Fix incorrect args to
+	_pthread_handler_pop_all() calls.
+	Make thread entry removal conditional.
+
+	* sync.c (pthread_join): Add multiple join and async detach handling.
+
+	* implement.h (_PTHREAD_THREADS_TABLE_INDEX): Add.
+
+	* global.c (_pthread_threads_mutex_table): Add.
+
+	* implement.h (_pthread_once_flag): Remove.
+	(_pthread_once_lock): Ditto.
+	(_pthread_threads_mutex_table): Add.
+
+	* global.c (_pthread_once_flag): Remove.
+	(_pthread_once_lock): Ditto.
+
+	* sync.c (pthread_join): Fix tests involving new return value
+	from _pthread_find_thread_entry().
+	(pthread_detach): Ditto.
+
+	* private.c (_pthread_find_thread_entry): Failure return code
+	changed from -1 to NULL.
+
 Fri Jul 24 23:09:33 1998  Ross Johnson  <rpj@ixobrychus.canberra.edu.au>
 
 	* create.c (pthread_create): Change . to -> in sigmask memcpy() args.
diff --git a/attr.c b/attr.c
index 22332c0..65c00a0 100644
--- a/attr.c
+++ b/attr.c
@@ -98,9 +98,14 @@ pthread_attr_init(pthread_attr_t *attr)
   attr->stacksize = PTHREAD_STACK_MIN;
 #endif
 
-  attr->cancelability = PTHREAD_CANCEL_ENABLE;
+  attr->cancelstate = PTHREAD_CANCEL_ENABLE;
   attr->canceltype = PTHREAD_CANCEL_DEFERRED;
-  attr->detached = PTHREAD_CREATE_JOINABLE;
+  attr->detachedstate = PTHREAD_CREATE_JOINABLE;
+  memset(&(attr->sigmask), 0, sizeof(sigset_t));
+
+  /* Priority uses Win32 priority values. */
+  int priority = THREAD_PRIORITY_NORMAL;
+
   attr->valid = 0;
 
   return 0;
diff --git a/create.c b/create.c
index 1d2303e..2a94baf 100644
--- a/create.c
+++ b/create.c
@@ -14,30 +14,35 @@
 #include "implement.h"
 
 unsigned
-_pthread_start_call(void * thisarg)
+_pthread_start_call(void * us_arg)
 {
   /* We're now in a running thread. Any local variables here are on
      this threads private stack so we're safe to leave data in them
      until we leave. */
-  _pthread_threads_thread__t * this;
+  _pthread_threads_thread__t * us;
+  pthread_mutex_t * us_thread_mutex;
   _pthread_call_t * call;
   unsigned (*func)(void *);
   void * arg;
   unsigned ret;
   int from;
 
-  this = (_pthread_threads_thread__t *) thisarg;
+  us = (_pthread_threads_thread__t *) us_arg;
 
-  if (this->detached == PTHREAD_CREATE_DETACHED)
+  /* FIXME: For now, if priority setting fails then at least ensure
+     that our records reflect true reality. */
+  if (SetThreadPriority((HANDLE) us->thread, us->attr.priority) == FALSE)
     {
-      (void) CloseHandle(this->thread);
+      us->attr.priority = GetThreadPriority((HANDLE) us->thread);
     }
 
-  func = this->call.routine;
-  arg = this->call.arg;
+  func = us->call.routine;
+  arg = us->call.arg;
+
+  us_thread_mutex = _PTHREAD_THREAD_MUTEX(us);
 
   /* FIXME: Should we be using sigsetjmp() here instead. */
-  from = setjmp(this->call.env);
+  from = setjmp(us->call.env);
 
   if (from == 0)
     {
@@ -45,6 +50,21 @@ _pthread_start_call(void * thisarg)
       ret = (*func)(arg);
 
       _pthread_vacuum();
+
+      /* Remove the thread entry on exit only if pthread_detach()
+	 was called and there are no waiting joins. */
+
+      /* CRITICAL SECTION */
+      pthread_mutex_lock(us_thread_mutex);
+
+      if (us->detach == TRUE
+	  && us->join_count == 0)
+	{
+	  _pthread_delete_thread_entry(us);
+	}
+
+      pthread_mutex_lock(us_thread_mutex);
+      /* END CRITICAL SECTION */
     }
   else
     {
@@ -52,10 +72,26 @@ _pthread_start_call(void * thisarg)
 	 func() called pthread_exit() which called longjmp(). */
       _pthread_vacuum();
 
-      /* Never returns. */
-      _endthreadex(0);
+      /* Remove the thread entry on exit only if pthread_detach()
+	 was called and there are no waiting joins. */
+
+      /* CRITICAL SECTION */
+      pthread_mutex_lock(us_thread_mutex);
+
+      if (us->detach == TRUE
+	  && us->join_count == 0)
+	{
+	  _pthread_delete_thread_entry(us);
+	}
+
+      pthread_mutex_lock(us_thread_mutex);
+      /* END CRITICAL SECTION */
+
+      ret = 0;
     }
 
+  /* From Win32's point of view we always return naturally from our
+     start routine and so it should clean up it's own thread residue. */
   return ret;
 }
 
@@ -78,9 +114,9 @@ pthread_create(pthread_t *thread,
     {
       attr_copy = &(this->attr);
 
+      /* Map given attributes otherwise just use default values. */
       if (attr != NULL) 
 	{
-	  /* Map attributes. */
 	  if (attr_copy->stacksize == 0)
 	    {
 	      attr_copy->stacksize = PTHREAD_STACK_MIN;
@@ -88,12 +124,14 @@ pthread_create(pthread_t *thread,
 
 	  attr_copy->cancelstate = attr->cancelstate;
 	  attr_copy->canceltype = attr->canceltype;
-	  attr_copy->detached = attr->detached;
+	  attr_copy->detachedstate = attr->detachedstate;
 	  attr_copy->priority = attr->priority;
 
 #if HAVE_SIGSET_T
 	  memcpy(&(attr_copy->sigmask), &(attr->sigmask), sizeof(sigset_t)); 
 #endif /* HAVE_SIGSET_T */
+
+	  this->detach = (attr->detachedstate == PTHREAD_CREATE_DETACHED);
 	}
 
       /* Start running, not suspended. */
diff --git a/exit.c b/exit.c
index a32e0ca..70fc7f2 100644
--- a/exit.c
+++ b/exit.c
@@ -6,31 +6,28 @@
  * a thread.
  */
 
-#include <windows.h>
-#include <process.h>
 #include "pthread.h"
 #include "implement.h"
 
 void
 _pthread_vacuum(void)
 {
-  /* This function can be called from pthread_exit(), or from
-     _pthread_start_call() in which case cleanupstack should be
-     empty but destructorstack still needs to be run. */
-  _pthread_threads_thread_t * this;
+  /* Run all the handlers. */
+  _pthread_handler_pop_all(_PTHREAD_CLEANUP_STACK, 
+			   _PTHREAD_HANDLER_EXECUTE);
 
-  this = *_PTHREAD_THIS;
+  _pthread_handler_pop_all(_PTHREAD_DESTRUCTOR_STACK, 
+			   _PTHREAD_HANDLER_EXECUTE);
 
-  /* Run all the handlers. */
-  _pthread_handler_pop_all(&(this->cleanupstack), _PTHREAD_HANDLER_EXECUTE);
-  _pthread_handler_pop_all(&(this->destructorstack), _PTHREAD_HANDLER_EXECUTE);
+  /* Pop any atfork handlers without executing them. */
+  _pthread_handler_pop_all(_PTHREAD_FORKPREPARE_STACK, 
+			   _PTHREAD_HANDLER_NOEXECUTE);
 
-  /* Pop any atfork handlers to free storage. */
-  _pthread_handler_pop_all(&(this->forkprepare), _PTHREAD_HANDLER_NOEXECUTE);
-  _pthread_handler_pop_all(&(this->forkparent), _PTHREAD_HANDLER_NOEXECUTE);
-  _pthread_handler_pop_all(&(this->forkchild), _PTHREAD_HANDLER_NOEXECUTE);
+  _pthread_handler_pop_all(_PTHREAD_FORKPARENT_STACK, 
+			   _PTHREAD_HANDLER_NOEXECUTE);
 
-  _pthread_delete_thread_entry(NULL);
+  _pthread_handler_pop_all(_PTHREAD_FORKCHILD_STACK,
+			   _PTHREAD_HANDLER_NOEXECUTE);
 }
 
 void
@@ -40,14 +37,13 @@ pthread_exit(void * value)
 
   this = _PTHREAD_THIS;
 
+  /* Copy value into the thread entry so it can be given
+     to any joining threads. */
   if (this->joinvalueptr != NULL)
     {
-      *(this->joinvalueptr) = value;
+      this->joinvalueptr = value;
     }
 
-  /* FIXME: More to do here. IE, if pthread_detach() was called
-     and value != NULL, do we free(value)? */
-
   /* Teleport back to _pthread_start_call() to cleanup and exit. */
   longjmp(this->call.env, 1);
 }
diff --git a/global.c b/global.c
index bcc034d..cbc300d 100644
--- a/global.c
+++ b/global.c
@@ -15,8 +15,8 @@ pthread_mutex_t _pthread_count_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 DWORD _pthread_threads_count = 0;
 
+/* Per thread management storage. */
 _pthread_threads_thread_t _pthread_threads_table[PTHREAD_THREADS_MAX];
 
-unsigned short _pthread_once_flag;
-
-pthread_mutex_t _pthread_once_lock = PTHREAD_MUTEX_INITIALIZER;
+/* Per thread mutex locks. */
+pthread_mutex_t _pthread_threads_mutex_table[PTHREAD_THREADS_MAX];
diff --git a/implement.h b/implement.h
index 033e0e8..807bf0e 100644
--- a/implement.h
+++ b/implement.h
@@ -38,11 +38,23 @@ typedef struct {
   jmpbuf env;
 } _pthread_call_t;
 
+/* Macro to return the address of the thread entry of the calling thread. */
 #define _PTHREAD_THIS (_pthread_find_thread_entry(pthread_this()))
 
+/* Macro to compute the address of a given handler stack. */
 #define _PTHREAD_STACK(stack) \
   ((_pthread_handler_node_t *) &(_PTHREAD_THIS)->cleanupstack + stack);
 
+/* Macro to compute the table index of a thread entry from it's entry
+   address. */
+#define _PTHREAD_THREADS_TABLE_INDEX(this) \
+  ((_pthread_threads_table_t *) this - \
+   (_pthread_threads_table_t *) _pthread_threads_threads_table)
+
+/* Macro to compute the address of a per-thread mutex lock. */
+#define _PTHREAD_THREAD_MUTEX(this) \
+   (&_pthread_threads_mutex_table[_PTHREAD_THREADS_TABLE_INDEX(this)])
+
 /* An element in the thread table. */
 typedef struct _pthread_threads_thread _pthread_threads_thread_t;
 
@@ -52,6 +64,7 @@ struct _pthread_threads_thread {
   _pthread_call_t             call;
   int                         cancelthread;
   void **                     joinvalueptr;
+  int                         join_count;
   _pthread_handler_node_t *   cleanupstack;
   _pthread_handler_node_t *   destructorstack;
   _pthread_handler_node_t *   forkpreparestack;
@@ -102,10 +115,7 @@ extern DWORD _pthread_threads_count;
 
 extern _pthread_threads_thread_t _pthread_threads_table[];
 
-extern unsigned short _pthread_once_flag;
-
-extern pthread_mutex_t _pthread_once_lock;
-
+extern pthread_mutex_t _pthread_threads_mutex_table[];
 
 #endif /* _IMPLEMENT_H */
 
diff --git a/private.c b/private.c
index dc7e111..d77f0b2 100644
--- a/private.c
+++ b/private.c
@@ -98,35 +98,27 @@ _pthread_find_thread_entry(pthread_t thread)
   if (this->thread == NULL || this == start)
     {
       /* Failed to find the thread. */
-      return -1;
+      return NULL;
     }
 
   return this;
 }
 
 void
-_pthread_delete_thread_entry(_pthread_threads_thread_t * this)
+_pthread_delete_thread_entry(_pthread_threads_thread_t * thread_entry)
 {
   /* We don't check that the thread has been properly cleaned up, so
      it had better be done already. */
-  _pthread_threads_thread_t * this;
-  _pthread_threads_thread_t * entry;
 
   /* CRITICAL SECTION */
   pthread_mutex_lock(&_pthread_count_mutex);
 
-  /* If this is not NULL then we are removing an entry for a
-     failed thread start. If this is NULL we need to get this
-     here within the critical section. */
-  if (this == NULL)
-    {
-      this = _PTHREAD_THIS;
-    }
+  /* Remove the thread entry if necessary. */
 
-  if (this->thread != NULL)
+  if (thread_entry->thread != NULL)
     {
-      this->thread = NULL;
-
+      thread_entry->thread = NULL;
+      
       if (_pthread_threads_count > 0)
 	{
 	  _pthread_threads_count--;
diff --git a/pthread.h b/pthread.h
index fa0c3f8..0c0036b 100644
--- a/pthread.h
+++ b/pthread.h
@@ -91,7 +91,7 @@ typedef struct {
   int cancelstate;                   /* PTHREAD_CANCEL_DISABLE
 					PTHREAD_CANCEL_ENABLE */
 
-  int detached;                      /* PTHREAD_CREATE_DETACHED
+  int detachedstate;                 /* PTHREAD_CREATE_DETACHED
 					PTHREAD_CREATE_JOINABLE */
 
   int canceltype;                    /* PTHREAD_CANCEL_ASYNCHRONOUS
diff --git a/sync.c b/sync.c
index c4e4327..84d9bd9 100644
--- a/sync.c
+++ b/sync.c
@@ -6,63 +6,147 @@
  * synchronisation.
  */
 
+/* POSIX STANDARD: A thread may pass a value pointer to some data via
+   pthread_exit(). That pointer will be stored in a location supplied
+   as an argument to pthread_join().
+
+   IMPLEMENTATION: The value_ptr is stored in the thread entry. When
+   pthread_join() wakes up after waiting, or immediately if the target
+   thread has already terminated but is not detached, the value
+   pointer from pthread_exit() will be copied to *value_ptr.
+
+   If the target thread does not become detached in the mean time, all
+   waiting joins on that thread will get the value pointer. The last
+   waiting join will delete the target thread entry.
+
+   ----
+
+   POSIX STANDARD: The results of multiple simultaneous calls to
+   pthread_join() specifying the same target thread are undefined.
+
+   IMPLEMENTATION: Any such join that occurs before the first such
+   join wakes up, or the thread is otherwise detached (by a call to
+   pthread_detach), will return successfully with the value that was
+   passed to pthread_exit(). After the last such join returns, the
+   target thread will have be detached and it's entry removed from the
+   thread table.
+  
+   Until the target thread entry is deleted it will be counted against
+   {PTHREAD_COUNT_MAX}.
+
+   ----
+
+   ----
+
+   POSIX STANDARD: It is unspecified whether a thread that has exited
+   but remains unjoined counts against {PTHREAD_COUNT_MAX}.
+
+   IMPLEMENTATION: A thread that has exited but remains unjoined will
+   be counted against {PTHREAD_COUNT_MAX}. The first call to
+   pthread_join() or pthread_detach() will remove the target thread's
+   table entry and decrement the count.
+
+   ---- */
+
+#include <windows.h>
 #include "pthread.h"
+#include "implement.h"
 
 int
 pthread_join(pthread_t thread, void ** valueptr)
 {
   LPDWORD exitcode;
   int detachstate;
-  pthread_t us = pthread_self();
+  _pthread_threads_thread_t * target;
 
   /* First check if we are trying to join to ourselves. */
-  if (pthread_equal(thread, us) == 0)
+  if (thread == pthread_self())
     {
       return EDEADLK;
     }
 
   /* Find the thread. */
-  this = _pthread_find_thread_entry(thread);
+  target = _pthread_find_thread_entry(thread);
 
-  if (this == -1)
+  if (target != NULL)
     {
-      return ESRCH;
-    }
+      pthread_mutex_t * target_thread_mutex;
+      int ret;
 
-  /* If the thread is detached, then join will return immediately. */
+      target_thread_mutex = _PTHREAD_THREAD_MUTEX(target);
 
-  if (pthread_attr_getdetachedstate(&(this->attr), &detachstate) != 0 
-      || detachstate == PTHREAD_CREATE_DETACHED)
-    {
-      return EINVAL;
-    }
+      /* CRITICAL SECTION */
+      pthread_mutex_lock(target_thread_mutex);
 
-  this->joinvalueptr = valueptr;
+      /* If the thread is in DETACHED state, then join will return
+	 immediately. */
 
-  /* Wait on the kernel thread object. */
-  switch (WaitForSingleObject(thread, INFINITE))
-    {
-    case WAIT_FAILED:
-      /* The thread does not exist. */
-      return ESRCH;
-    case WAIT_OBJECT_0:
-      /* The thread has finished. */
-      break;
-    default:
-      /* This should never happen. */
-      break;
-    }
- 
-  /* We don't get the exit code as a result of the last operation,
-     so we do it now. */
+      if (target->detach == TRUE)
+	{
+	  return EINVAL;
+	}
 
-  if (GetExitCodeThread(thread, exitcode) != TRUE)
-    {
-      return ESRCH;
+      target->join_count++;
+
+      pthread_mutex_lock(target_thread_mutex);
+      /* END CRITICAL SECTION */
+
+      /* Wait on the kernel thread object. */
+      switch (WaitForSingleObject(thread, INFINITE))
+	{
+	case WAIT_FAILED:
+	  /* The thread does not exist. */
+	  return ESRCH;
+	case WAIT_OBJECT_0:
+	  /* The thread has finished. */
+	  break;
+	default:
+	  /* This should never happen. */
+	  break;
+	}
+
+      /* We know the target thread entry still exists at this point
+	 because we incremented join_count above after checking. The
+	 thread entry will not be removed until join_count == 0 again,
+	 ie. when the last waiting join has passed through the
+	 following critical section. */
+
+      /* CRITICAL SECTION */
+      pthread_mutex_lock(target_thread_mutex);
+
+      /* Collect the value pointer passed to pthread_exit().  If
+	 another thread detaches our target thread while we're
+	 waiting, then we report a deadlock as it likely that storage
+	 pointed to by target->joinvalueptr has been freed or
+	 otherwise no longer valid. */
+
+      if (target->detach == TRUE)
+	{
+	  ret = EDEADLK;
+	}
+      else
+	{
+	  *value_ptr = target->joinvalueptr;
+	  ret = 0;
+	}
+
+      target->join_count--;
+
+      /* If we're the last join to return then we are responsible for
+	 removing the target thread's table entry. */
+      if (target->join_count == 0)
+	{
+	  _pthread_delete_thread_entry(target);
+	}
+
+      pthread_mutex_lock(target_thread_mutex);
+      /* END CRITICAL SECTION */
+
+      return ret;
     }
 
-  /* FIXME: this is wrong. */
-  return &exitcode;
+  /* Thread not found. */
+  return ESRCH;
 }
 
 int
@@ -73,7 +157,7 @@ pthread_detach(pthread_t thread)
 
   this = _pthread_find_thread_entry(thread);
 
-  if (this == -1)
+  if (this == NULL)
     {
       return ESRCH;
     }
@@ -84,11 +168,10 @@ pthread_detach(pthread_t thread)
     {
       return EINVAL;
     }
-  
-  this->attr.detached = PTHREAD_CREATE_DETACHED;
-  if (CloseHandle(thread) != TRUE)
-    {
-      return ESRCH;
-    }
+
+  /* This is all we do here - the rest is done either when the thread
+     exits or when pthread_join() exits. */
+  this->detach = TRUE;
+
   return 0;
 }