thread_local_alloc.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. /*
  2. * Copyright (c) 2000-2005 by Hewlett-Packard Company. All rights reserved.
  3. *
  4. * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
  5. * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
  6. *
  7. * Permission is hereby granted to use or copy this program
  8. * for any purpose, provided the above notices are retained on all copies.
  9. * Permission to modify the code and to distribute modified code is granted,
  10. * provided the above notices are retained, and a notice that the code was
  11. * modified is included with the above copyright notice.
  12. */
  13. #include "private/gc_priv.h"
  14. #if defined(THREAD_LOCAL_ALLOC)
  15. #ifndef THREADS
  16. # error "invalid config - THREAD_LOCAL_ALLOC requires GC_THREADS"
  17. #endif
  18. #include "private/thread_local_alloc.h"
  19. #include <stdlib.h>
  20. #if defined(USE_COMPILER_TLS)
  21. __thread GC_ATTR_TLS_FAST
  22. #elif defined(USE_WIN32_COMPILER_TLS)
  23. __declspec(thread) GC_ATTR_TLS_FAST
  24. #endif
  25. GC_key_t GC_thread_key;
  26. static GC_bool keys_initialized;
  27. /* Return a single nonempty freelist fl to the global one pointed to */
  28. /* by gfl. */
  29. static void return_single_freelist(void *fl, void **gfl)
  30. {
  31. if (*gfl == 0) {
  32. *gfl = fl;
  33. } else {
  34. void *q, **qptr;
  35. GC_ASSERT(GC_size(fl) == GC_size(*gfl));
  36. /* Concatenate: */
  37. qptr = &(obj_link(fl));
  38. while ((word)(q = *qptr) >= HBLKSIZE)
  39. qptr = &(obj_link(q));
  40. GC_ASSERT(0 == q);
  41. *qptr = *gfl;
  42. *gfl = fl;
  43. }
  44. }
  45. /* Recover the contents of the freelist array fl into the global one gfl.*/
  46. /* We hold the allocator lock. */
  47. static void return_freelists(void **fl, void **gfl)
  48. {
  49. int i;
  50. for (i = 1; i < TINY_FREELISTS; ++i) {
  51. if ((word)(fl[i]) >= HBLKSIZE) {
  52. return_single_freelist(fl[i], &gfl[i]);
  53. }
  54. /* Clear fl[i], since the thread structure may hang around. */
  55. /* Do it in a way that is likely to trap if we access it. */
  56. fl[i] = (ptr_t)HBLKSIZE;
  57. }
  58. /* The 0 granule freelist really contains 1 granule objects. */
  59. # ifdef GC_GCJ_SUPPORT
  60. if (fl[0] == ERROR_FL) return;
  61. # endif
  62. if ((word)(fl[0]) >= HBLKSIZE) {
  63. return_single_freelist(fl[0], &gfl[1]);
  64. }
  65. }
  66. #ifdef USE_PTHREAD_SPECIFIC
  67. /* Re-set the TLS value on thread cleanup to allow thread-local */
  68. /* allocations to happen in the TLS destructors. */
  69. /* GC_unregister_my_thread (and similar routines) will finally set */
  70. /* the GC_thread_key to NULL preventing this destructor from being */
  71. /* called repeatedly. */
  72. static void reset_thread_key(void* v) {
  73. pthread_setspecific(GC_thread_key, v);
  74. }
  75. #else
  76. # define reset_thread_key 0
  77. #endif
  78. /* Each thread structure must be initialized. */
  79. /* This call must be made from the new thread. */
  80. GC_INNER void GC_init_thread_local(GC_tlfs p)
  81. {
  82. int i, j, res;
  83. GC_ASSERT(I_HOLD_LOCK());
  84. if (!EXPECT(keys_initialized, TRUE)) {
  85. GC_ASSERT((word)&GC_thread_key % sizeof(word) == 0);
  86. res = GC_key_create(&GC_thread_key, reset_thread_key);
  87. if (COVERT_DATAFLOW(res) != 0) {
  88. ABORT("Failed to create key for local allocator");
  89. }
  90. keys_initialized = TRUE;
  91. }
  92. res = GC_setspecific(GC_thread_key, p);
  93. if (COVERT_DATAFLOW(res) != 0) {
  94. ABORT("Failed to set thread specific allocation pointers");
  95. }
  96. for (j = 0; j < TINY_FREELISTS; ++j) {
  97. for (i = 0; i < THREAD_FREELISTS_KINDS; ++i) {
  98. p -> _freelists[i][j] = (void *)(word)1;
  99. }
  100. # ifdef GC_GCJ_SUPPORT
  101. p -> gcj_freelists[j] = (void *)(word)1;
  102. # endif
  103. }
  104. /* The size 0 free lists are handled like the regular free lists, */
  105. /* to ensure that the explicit deallocation works. However, */
  106. /* allocation of a size 0 "gcj" object is always an error. */
  107. # ifdef GC_GCJ_SUPPORT
  108. p -> gcj_freelists[0] = ERROR_FL;
  109. # endif
  110. }
  111. /* We hold the allocator lock. */
  112. GC_INNER void GC_destroy_thread_local(GC_tlfs p)
  113. {
  114. int k;
  115. /* We currently only do this from the thread itself. */
  116. GC_STATIC_ASSERT(THREAD_FREELISTS_KINDS <= MAXOBJKINDS);
  117. for (k = 0; k < THREAD_FREELISTS_KINDS; ++k) {
  118. if (k == (int)GC_n_kinds)
  119. break; /* kind is not created */
  120. return_freelists(p -> _freelists[k], GC_obj_kinds[k].ok_freelist);
  121. }
  122. # ifdef GC_GCJ_SUPPORT
  123. return_freelists(p -> gcj_freelists, (void **)GC_gcjobjfreelist);
  124. # endif
  125. }
  126. GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_kind(size_t bytes, int kind)
  127. {
  128. size_t granules;
  129. void *tsd;
  130. void *result;
  131. # if MAXOBJKINDS > THREAD_FREELISTS_KINDS
  132. if (EXPECT(kind >= THREAD_FREELISTS_KINDS, FALSE)) {
  133. return GC_malloc_kind_global(bytes, kind);
  134. }
  135. # endif
  136. # if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
  137. {
  138. GC_key_t k = GC_thread_key;
  139. if (EXPECT(0 == k, FALSE)) {
  140. /* We haven't yet run GC_init_parallel. That means */
  141. /* we also aren't locking, so this is fairly cheap. */
  142. return GC_malloc_kind_global(bytes, kind);
  143. }
  144. tsd = GC_getspecific(k);
  145. }
  146. # else
  147. if (!EXPECT(keys_initialized, TRUE))
  148. return GC_malloc_kind_global(bytes, kind);
  149. tsd = GC_getspecific(GC_thread_key);
  150. # endif
  151. # if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS)
  152. if (EXPECT(0 == tsd, FALSE)) {
  153. return GC_malloc_kind_global(bytes, kind);
  154. }
  155. # endif
  156. GC_ASSERT(GC_is_initialized);
  157. GC_ASSERT(GC_is_thread_tsd_valid(tsd));
  158. granules = ROUNDED_UP_GRANULES(bytes);
  159. GC_FAST_MALLOC_GRANS(result, granules,
  160. ((GC_tlfs)tsd) -> _freelists[kind], DIRECT_GRANULES,
  161. kind, GC_malloc_kind_global(bytes, kind),
  162. (void)(kind == PTRFREE ? NULL
  163. : (obj_link(result) = 0)));
  164. # ifdef LOG_ALLOCS
  165. GC_log_printf("GC_malloc_kind(%lu, %d) returned %p, recent GC #%lu\n",
  166. (unsigned long)bytes, kind, result,
  167. (unsigned long)GC_gc_no);
  168. # endif
  169. return result;
  170. }
  171. #ifdef GC_GCJ_SUPPORT
  172. # include "gc_gcj.h"
  173. /* Gcj-style allocation without locks is extremely tricky. The */
  174. /* fundamental issue is that we may end up marking a free list, which */
  175. /* has freelist links instead of "vtable" pointers. That is usually */
  176. /* OK, since the next object on the free list will be cleared, and */
  177. /* will thus be interpreted as containing a zero descriptor. That's */
  178. /* fine if the object has not yet been initialized. But there are */
  179. /* interesting potential races. */
  180. /* In the case of incremental collection, this seems hopeless, since */
  181. /* the marker may run asynchronously, and may pick up the pointer to */
  182. /* the next freelist entry (which it thinks is a vtable pointer), get */
  183. /* suspended for a while, and then see an allocated object instead */
  184. /* of the vtable. This may be avoidable with either a handshake with */
  185. /* the collector or, probably more easily, by moving the free list */
  186. /* links to the second word of each object. The latter isn't a */
  187. /* universal win, since on architecture like Itanium, nonzero offsets */
  188. /* are not necessarily free. And there may be cache fill order issues. */
  189. /* For now, we punt with incremental GC. This probably means that */
  190. /* incremental GC should be enabled before we fork a second thread. */
  191. /* Unlike the other thread local allocation calls, we assume that the */
  192. /* collector has been explicitly initialized. */
  193. GC_API GC_ATTR_MALLOC void * GC_CALL GC_gcj_malloc(size_t bytes,
  194. void * ptr_to_struct_containing_descr)
  195. {
  196. if (EXPECT(GC_incremental, FALSE)) {
  197. return GC_core_gcj_malloc(bytes, ptr_to_struct_containing_descr);
  198. } else {
  199. size_t granules = ROUNDED_UP_GRANULES(bytes);
  200. void *result;
  201. void **tiny_fl;
  202. GC_ASSERT(GC_gcj_malloc_initialized);
  203. tiny_fl = ((GC_tlfs)GC_getspecific(GC_thread_key))->gcj_freelists;
  204. GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES,
  205. GC_gcj_kind,
  206. GC_core_gcj_malloc(bytes,
  207. ptr_to_struct_containing_descr),
  208. {AO_compiler_barrier();
  209. *(void **)result = ptr_to_struct_containing_descr;});
  210. /* This forces the initialization of the "method ptr". */
  211. /* This is necessary to ensure some very subtle properties */
  212. /* required if a GC is run in the middle of such an allocation. */
  213. /* Here we implicitly also assume atomicity for the free list. */
  214. /* and method pointer assignments. */
  215. /* We must update the freelist before we store the pointer. */
  216. /* Otherwise a GC at this point would see a corrupted */
  217. /* free list. */
  218. /* A real memory barrier is not needed, since the */
  219. /* action of stopping this thread will cause prior writes */
  220. /* to complete. */
  221. /* We assert that any concurrent marker will stop us. */
  222. /* Thus it is impossible for a mark procedure to see the */
  223. /* allocation of the next object, but to see this object */
  224. /* still containing a free list pointer. Otherwise the */
  225. /* marker, by misinterpreting the freelist link as a vtable */
  226. /* pointer, might find a random "mark descriptor" in the next */
  227. /* object. */
  228. return result;
  229. }
  230. }
  231. #endif /* GC_GCJ_SUPPORT */
  232. /* The thread support layer must arrange to mark thread-local */
  233. /* free lists explicitly, since the link field is often */
  234. /* invisible to the marker. It knows how to find all threads; */
  235. /* we take care of an individual thread freelist structure. */
  236. GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p)
  237. {
  238. ptr_t q;
  239. int i, j;
  240. for (j = 0; j < TINY_FREELISTS; ++j) {
  241. for (i = 0; i < THREAD_FREELISTS_KINDS; ++i) {
  242. /* Load the pointer atomically as it might be updated */
  243. /* concurrently by GC_FAST_MALLOC_GRANS. */
  244. q = (ptr_t)AO_load((volatile AO_t *)&p->_freelists[i][j]);
  245. if ((word)q > HBLKSIZE)
  246. GC_set_fl_marks(q);
  247. }
  248. # ifdef GC_GCJ_SUPPORT
  249. if (EXPECT(j > 0, TRUE)) {
  250. q = (ptr_t)AO_load((volatile AO_t *)&p->gcj_freelists[j]);
  251. if ((word)q > HBLKSIZE)
  252. GC_set_fl_marks(q);
  253. }
  254. # endif
  255. }
  256. }
  257. #if defined(GC_ASSERTIONS)
  258. /* Check that all thread-local free-lists in p are completely marked. */
  259. void GC_check_tls_for(GC_tlfs p)
  260. {
  261. int i, j;
  262. for (j = 1; j < TINY_FREELISTS; ++j) {
  263. for (i = 0; i < THREAD_FREELISTS_KINDS; ++i) {
  264. GC_check_fl_marks(&p->_freelists[i][j]);
  265. }
  266. # ifdef GC_GCJ_SUPPORT
  267. GC_check_fl_marks(&p->gcj_freelists[j]);
  268. # endif
  269. }
  270. }
  271. #endif /* GC_ASSERTIONS */
  272. #endif /* THREAD_LOCAL_ALLOC */