Fix libc and pthread cleanup before PSP module teardown

src/libcglue/Makefile.am

@@ -34,10 +34,10 @@ GLUE_OBJS = __dummy_passwd.o __psp_heap_blockid.o __psp_free_heap.o _fork.o _wai
 
 INIT_OBJS = \
 	__libpthreadglue_init.o \
+	__libpthreadglue_deinit.o \
 	__libcglue_init.o \
 	__gprof_cleanup.o \
-	__libcglue_deinit.o \
-	exit.o
+	__libcglue_deinit.o
 
 LOCK_OBJS = \
 	__lock___sfp_recursive_mutex.o \

src/libcglue/init.c

@@ -13,6 +13,7 @@
  */
 
 #include <stdio.h>
+#include <stdlib.h>
 #include <unistd.h>
 #include <string.h>
 #include <sys/param.h>
@@ -24,6 +25,7 @@ void __timezone_update();
 void __fdman_init();
 void __init_mutex();
 void pthread_init();
+void pthread_terminate(void);
 void __psp_free_heap();
 void __deinit_mutex();
 void __locks_init();
@@ -52,6 +54,19 @@ void __libpthreadglue_init()
 void __libpthreadglue_init();
 #endif
 
+#if defined(F___libpthreadglue_deinit) && !defined(PSP_WITHOUT_PTHREAD)
+/* Match the initialization hook while keeping the heap and locks available. */
+__attribute__((weak))
+void __libpthreadglue_deinit(void)
+{
+    pthread_terminate();
+}
+#elif defined(PSP_WITHOUT_PTHREAD)
+#define __libpthreadglue_deinit() do{}while(0)
+#else
+void __libpthreadglue_deinit(void);
+#endif
+
 #ifdef F___libcglue_init
 /* Note: This function is being called from crt0.c/crt0_prx.c.
 * It is a weak function because can be override by user program, 
@@ -73,6 +88,10 @@ void __libcglue_init(int argc, char *argv[])
 
 	/* Initialize pthread library */
 	__libpthreadglue_init();
+#ifndef PSP_WITHOUT_PTHREAD
+    /* Run TLS cleanup after C++ finalizers, while stdio and the heap live. */
+    atexit(__libpthreadglue_deinit);
+#endif
 
 	/* Initialize cwd from this program's path */
 	__init_cwd(argv[0]);
@@ -90,18 +109,10 @@ __attribute__((weak))
 void __libcglue_deinit()
 {
     __gprof_cleanup();
-    __psp_free_heap();
 	__deinit_mutex();
 	__locks_deinit();
+    __psp_free_heap();
 }
 #else
 void __libcglue_deinit();
 #endif
-
-#ifdef F_exit
-__attribute__((weak))
-void exit(int retval)
-{
-	_exit(retval);
-}
-#endif

src/libpthreadglue/Makefile.am

@@ -15,7 +15,7 @@ CPPFLAGS = -I$(top_srcdir)/src/base \
 CFLAGS = @PSPSDK_CFLAGS@
 CCASFLAGS = $(CFLAGS)
 
-OSAL_OBJS = __threadDataKey.o __getThreadData.o __pspStubThreadEntry.o \
+OSAL_OBJS = __threadDataKey.o __semaphoreRegistry.o __getThreadData.o __pspStubThreadEntry.o \
 	pte_osInit.o pte_osTerminate.o pte_osThreadCreate.o pte_osThreadStart.o pte_osThreadDelete.o \
 	pte_osThreadExitAndDelete.o pte_osThreadExit.o pte_osThreadWaitForEnd.o pte_osThreadGetHandle.o pte_osThreadGetPriority.o pthread_num_processors_np.o \
 	pte_osThreadSetPriority.o pte_osThreadCancel.o pte_osThreadCheckCancel.o pte_osThreadSleep.o pte_osThreadGetMinPriority.o pte_osThreadGetMaxPriority.o \
@@ -39,4 +39,4 @@ $(OSAL_OBJS): osal.c
 	$(AM_V_CC)$(COMPILE) -DF_$* $< -c -o $@
 
 $(TLS_HELPER_OBJS): tls-helper.c
-	$(AM_V_CC)$(COMPILE) -DF_$* $< -c -o $@
+	$(AM_V_CC)$(COMPILE) -DF_$* $< -c -o $@

src/libpthreadglue/osal.c

@@ -57,6 +57,26 @@ extern unsigned int __threadDataKey;
 
 extern void *__globalTls;
 
+/* Each linked runtime owns only the semaphores created through its OSAL. */
+typedef struct pspSemaphoreNode
+  {
+    pte_osSemaphoreHandle handle;
+    struct pspSemaphoreNode *next;
+  } pspSemaphoreNode;
+
+typedef struct pspSemaphoreRegistry
+  {
+    SceLwMutexWorkarea lock;
+    pspSemaphoreNode *head;
+    int initialized;
+  } pspSemaphoreRegistry;
+
+#ifdef F___semaphoreRegistry
+pspSemaphoreRegistry __semaphoreRegistry;
+#else
+extern pspSemaphoreRegistry __semaphoreRegistry;
+#endif
+
 /*
  * Data stored on a per-thread basis - allocated in pte_osThreadCreate
  * and freed in pte_osThreadDelete.
@@ -128,6 +148,15 @@ pte_osResult pte_osInit(void)
   result = pteTlsGlobalInit(PSP_MAX_TLS);
 
   if (result == PTE_OS_OK) {
+    /* Use a native lock: creating a pthread mutex would recurse into OSAL. */
+    if (sceKernelCreateLwMutex(&__semaphoreRegistry.lock,
+                              "pthread_semaphores", 0, 0, NULL) < 0) {
+      pteTlsGlobalDestroy();
+      return PTE_OS_NO_RESOURCES;
+    }
+    __semaphoreRegistry.head = NULL;
+    __semaphoreRegistry.initialized = 1;
+
     /* Allocate a key that we use to store control information (e.g. cancellation semaphore) per thread */
     result = __pteTlsAlloc(&__threadDataKey);
 
@@ -159,9 +188,13 @@ pte_osResult pte_osInit(void)
                               0,          /* initial value        */
                               255,        /* maximum value        */
                               0);         /* options (default)    */
-        result = PTE_OS_OK;
+        result = pThreadData->cancelSem < 0 ? PTE_OS_NO_RESOURCES : PTE_OS_OK;
       }
     }
+
+    if (result != PTE_OS_OK) {
+      pte_osTerminate();
+    }
   }
 
   return result;
@@ -170,6 +203,37 @@ pte_osResult pte_osInit(void)
 
 #ifdef F_pte_osTerminate
 pte_osResult pte_osTerminate(void) {
+  if (__semaphoreRegistry.initialized) {
+    pspSemaphoreNode *node;
+
+    /* Callers must finish all POSIX worker threads before terminating the runtime. */
+    sceKernelLockLwMutex(&__semaphoreRegistry.lock, 1, NULL);
+    node = __semaphoreRegistry.head;
+    __semaphoreRegistry.head = NULL;
+    sceKernelUnlockLwMutex(&__semaphoreRegistry.lock, 1);
+
+    while (node != NULL) {
+      pspSemaphoreNode *next = node->next;
+      sceKernelDeleteSema(node->handle);
+      free(node);
+      node = next;
+    }
+
+    sceKernelDeleteLwMutex(&__semaphoreRegistry.lock);
+    __semaphoreRegistry.initialized = 0;
+  }
+
+  if (__globalTls != NULL) {
+    pspThreadData *pThreadData = pteTlsGetValue(__globalTls, __threadDataKey);
+    if (pThreadData != NULL) {
+      if (pThreadData->cancelSem >= 0) {
+        sceKernelDeleteSema(pThreadData->cancelSem);
+      }
+      free(pThreadData);
+    }
+    pteTlsThreadDestroy(__globalTls);
+    __globalTls = NULL;
+  }
   pteTlsGlobalDestroy();
   return PTE_OS_OK;
 }
@@ -592,9 +656,14 @@ pte_osResult pte_osMutexUnlock(pte_osMutexHandle handle)
 pte_osResult pte_osSemaphoreCreate(int initialValue, pte_osSemaphoreHandle *pHandle)
 {
   pte_osSemaphoreHandle handle;
+  pspSemaphoreNode *node;
   static int semCtr = 0;
   char semName[32];
 
+  if (!__semaphoreRegistry.initialized) {
+    return PTE_OS_NO_RESOURCES;
+  }
+
   if (semCtr++ > MAX_PSP_UID) {
     semCtr = 0;
   }
@@ -610,6 +679,18 @@ pte_osResult pte_osSemaphoreCreate(int initialValue, pte_osSemaphoreHandle *pHan
     return PTE_OS_NO_RESOURCES;
   }
 
+  node = malloc(sizeof(*node));
+  if (node == NULL) {
+    sceKernelDeleteSema(handle);
+    return PTE_OS_NO_RESOURCES;
+  }
+
+  node->handle = handle;
+  sceKernelLockLwMutex(&__semaphoreRegistry.lock, 1, NULL);
+  node->next = __semaphoreRegistry.head;
+  __semaphoreRegistry.head = node;
+  sceKernelUnlockLwMutex(&__semaphoreRegistry.lock, 1);
+
   *pHandle = handle;
   return PTE_OS_OK;
 }
@@ -618,7 +699,23 @@ pte_osResult pte_osSemaphoreCreate(int initialValue, pte_osSemaphoreHandle *pHan
 #ifdef F_pte_osSemaphoreDelete
 pte_osResult pte_osSemaphoreDelete(pte_osSemaphoreHandle handle)
 {
+  pspSemaphoreNode **link;
+  pspSemaphoreNode *node = NULL;
+
+  if (__semaphoreRegistry.initialized) {
+    sceKernelLockLwMutex(&__semaphoreRegistry.lock, 1, NULL);
+    for (link = &__semaphoreRegistry.head; *link != NULL; link = &(*link)->next) {
+      if ((*link)->handle == handle) {
+        node = *link;
+        *link = node->next;
+        break;
+      }
+    }
+    sceKernelUnlockLwMutex(&__semaphoreRegistry.lock, 1);
+  }
+
   sceKernelDeleteSema(handle);
+  free(node);
   return PTE_OS_OK;
 }
 #endif

src/startup/crt0.c

@@ -80,6 +80,9 @@ void _main(SceSize args, void *argp)
 	/* Call libc initialization hook */
 	__libcglue_init(argc, argv);
 
+	/* Run finalizers before newlib closes streams and releases their locks. */
+	atexit(_fini);
+
 	/* Init can contain C++ constructors that require working threading */
 	_init();
 
@@ -93,9 +96,6 @@ void _main(SceSize args, void *argp)
 __attribute__((__noreturn__))
 void _exit(int status)
 {
-	/* call global destructors*/
-	_fini();
-
 	// uninitialize libcglue
 	__libcglue_deinit();
 

src/startup/crt0_prx.c

@@ -80,6 +80,9 @@ void _main(SceSize args, void *argp)
 	/* Call libc initialization hook */
 	__libcglue_init(argc, argv);	
 
+	/* Run finalizers before newlib closes streams and releases their locks. */
+	atexit(_fini);
+
 	/* Init can contain C++ constructors that require working threading */
 	_init();
 
@@ -93,9 +96,6 @@ void _main(SceSize args, void *argp)
 __attribute__((__noreturn__))
 void _exit(int status)
 {
-	/* call global destructors*/
-	_fini();
-
 	// uninitialize libcglue
 	__libcglue_deinit();