/* * Copyright (c) 1994 by Xerox Corporation. All rights reserved. * Copyright (c) 1996 by Silicon Graphics. All rights reserved. * Copyright (c) 1998 by Fergus Henderson. All rights reserved. * Copyright (c) 2000-2009 by Hewlett-Packard Development Company. * All rights reserved. * Copyright (c) 2009-2018 Ivan Maidanski * * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. */ #ifndef __cplusplus #define GC_INNER STATIC #define GC_EXTERN GC_INNER #endif #ifndef GC_DBG_MLC_H #define GC_DBG_MLC_H #ifndef GC_PRIVATE_H #define GC_PRIVATE_H #ifdef HAVE_CONFIG_H #include "config.h" #endif #if!defined(GC_BUILD)&&!defined(NOT_GCBUILD) #define GC_BUILD #endif #if (defined(__linux__)||defined(__GLIBC__)||defined(__GNU__)||(defined(__CYGWIN__)&&(defined(GC_THREADS)||!defined(USE_MMAP))))&&!defined(_GNU_SOURCE) #define _GNU_SOURCE 1 #endif #if defined(__INTERIX)&&!defined(_ALL_SOURCE) #define _ALL_SOURCE 1 #endif #if (defined(DGUX)&&defined(GC_THREADS)||defined(DGUX386_THREADS)||defined(GC_DGUX386_THREADS))&&!defined(_USING_POSIX4A_DRAFT10) #define _USING_POSIX4A_DRAFT10 1 #endif #if defined(__MINGW32__)&&!defined(__MINGW_EXCPT_DEFINE_PSDK)&&defined(__i386__)&&defined(GC_EXTERN) #define __MINGW_EXCPT_DEFINE_PSDK 1 #endif #if defined(NO_DEBUGGING)&&!defined(GC_ASSERTIONS)&&!defined(NDEBUG) #define NDEBUG 1 #endif #ifndef GC_H #ifndef GC_H #define GC_H #if (defined(WIN64)&&!defined(_WIN64))&&defined(_MSC_VER) #pragma message("Warning:Expecting _WIN64 for x64 targets!Notice the leading underscore!") #endif #if defined(GC_H) #define GC_TMP_VERSION_MAJOR 8 #define GC_TMP_VERSION_MINOR 1 #define GC_TMP_VERSION_MICRO 0 #ifdef GC_VERSION_MAJOR #if GC_TMP_VERSION_MAJOR!=GC_VERSION_MAJOR||GC_TMP_VERSION_MINOR!=GC_VERSION_MINOR||GC_TMP_VERSION_MICRO!=GC_VERSION_MICRO #error Inconsistent version info. Check README.md,include/gc_version.h and configure.ac. #endif #else #define GC_VERSION_MAJOR GC_TMP_VERSION_MAJOR #define GC_VERSION_MINOR GC_TMP_VERSION_MINOR #define GC_VERSION_MICRO GC_TMP_VERSION_MICRO #endif #endif #if defined(GC_H) #if defined(__GNUC__)&&defined(__GNUC_MINOR__) #define GC_GNUC_PREREQ(major,minor)((__GNUC__<<16)+__GNUC_MINOR__>=((major)<<16)+(minor)) #else #define GC_GNUC_PREREQ(major,minor)0 #endif #if defined(SOLARIS_THREADS)||defined(_SOLARIS_THREADS)||defined(_SOLARIS_PTHREADS)||defined(GC_SOLARIS_PTHREADS) #ifndef GC_SOLARIS_THREADS #define GC_SOLARIS_THREADS #endif #endif #if defined(IRIX_THREADS) #define GC_IRIX_THREADS #endif #if defined(DGUX_THREADS)&&!defined(GC_DGUX386_THREADS) #define GC_DGUX386_THREADS #endif #if defined(AIX_THREADS) #define GC_AIX_THREADS #endif #if defined(HPUX_THREADS) #define GC_HPUX_THREADS #endif #if defined(OSF1_THREADS) #define GC_OSF1_THREADS #endif #if defined(LINUX_THREADS) #define GC_LINUX_THREADS #endif #if defined(WIN32_THREADS) #define GC_WIN32_THREADS #endif #if defined(RTEMS_THREADS) #define GC_RTEMS_PTHREADS #endif #if defined(USE_LD_WRAP) #define GC_USE_LD_WRAP #endif #if defined(GC_WIN32_PTHREADS)&&!defined(GC_WIN32_THREADS) #define GC_WIN32_THREADS #endif #if defined(GC_AIX_THREADS)||defined(GC_DARWIN_THREADS)||defined(GC_DGUX386_THREADS)||defined(GC_FREEBSD_THREADS)||defined(GC_HPUX_THREADS)||defined(GC_IRIX_THREADS)||defined(GC_LINUX_THREADS)||defined(GC_NETBSD_THREADS)||defined(GC_OPENBSD_THREADS)||defined(GC_OSF1_THREADS)||defined(GC_SOLARIS_THREADS)||defined(GC_WIN32_THREADS)||defined(GC_RTEMS_PTHREADS) #ifndef GC_THREADS #define GC_THREADS #endif #elif defined(GC_THREADS) #if defined(__linux__) #define GC_LINUX_THREADS #elif defined(__OpenBSD__) #define GC_OPENBSD_THREADS #elif defined(_PA_RISC1_1)||defined(_PA_RISC2_0)||defined(hppa)||defined(__HPPA)||(defined(__ia64)&&defined(_HPUX_SOURCE)) #define GC_HPUX_THREADS #elif defined(__HAIKU__) #define GC_HAIKU_THREADS #elif defined(__DragonFly__)||defined(__FreeBSD_kernel__)||(defined(__FreeBSD__)&&!defined(SN_TARGET_ORBIS)) #define GC_FREEBSD_THREADS #elif defined(__NetBSD__) #define GC_NETBSD_THREADS #elif defined(__alpha)||defined(__alpha__) #define GC_OSF1_THREADS #elif (defined(mips)||defined(__mips)||defined(_mips))&&!(defined(nec_ews)||defined(_nec_ews)||defined(ultrix)||defined(__ultrix)) #define GC_IRIX_THREADS #elif defined(__sparc)||((defined(sun)||defined(__sun))&&(defined(i386)||defined(__i386__)||defined(__amd64)||defined(__amd64__))) #define GC_SOLARIS_THREADS #elif defined(__APPLE__)&&defined(__MACH__) #define GC_DARWIN_THREADS #endif #if defined(DGUX)&&(defined(i386)||defined(__i386__)) #define GC_DGUX386_THREADS #endif #if defined(_AIX) #define GC_AIX_THREADS #endif #if (defined(_WIN32)||defined(_MSC_VER)||defined(__BORLANDC__)||defined(__CYGWIN32__)||defined(__CYGWIN__)||defined(__CEGCC__)||defined(_WIN32_WCE)||defined(__MINGW32__))&&!defined(GC_WIN32_THREADS) #define GC_WIN32_THREADS #endif #if defined(__rtems__)&&(defined(i386)||defined(__i386__)) #define GC_RTEMS_PTHREADS #endif #endif #undef GC_PTHREADS #if (!defined(GC_WIN32_THREADS)||defined(GC_WIN32_PTHREADS)||defined(__CYGWIN32__)||defined(__CYGWIN__))&&defined(GC_THREADS)&&!defined(NN_PLATFORM_CTR)&&!defined(NN_BUILD_TARGET_PLATFORM_NX) #define GC_PTHREADS #endif #if!defined(_PTHREADS)&&defined(GC_NETBSD_THREADS) #define _PTHREADS #endif #if defined(GC_DGUX386_THREADS)&&!defined(_POSIX4A_DRAFT10_SOURCE) #define _POSIX4A_DRAFT10_SOURCE 1 #endif #if!defined(_REENTRANT)&&defined(GC_PTHREADS)&&!defined(GC_WIN32_THREADS) #define _REENTRANT 1 #endif #define __GC #if!defined(_WIN32_WCE)||defined(__GNUC__) #include #if defined(__MINGW32__)&&!defined(_WIN32_WCE) #include #endif #else #include #ifndef _PTRDIFF_T_DEFINED #define _PTRDIFF_T_DEFINED typedef long ptrdiff_t; #endif #endif #if!defined(GC_NOT_DLL)&&!defined(GC_DLL)&&((defined(_DLL)&&!defined(__GNUC__))||(defined(DLL_EXPORT)&&defined(GC_BUILD))) #define GC_DLL #endif #if defined(GC_DLL)&&!defined(GC_API) #if defined(__CEGCC__) #if defined(GC_BUILD) #define GC_API __declspec(dllexport) #else #define GC_API __declspec(dllimport) #endif #elif defined(__MINGW32__) #if defined(__cplusplus)&&defined(GC_BUILD) #define GC_API extern __declspec(dllexport) #elif defined(GC_BUILD)||defined(__MINGW32_DELAY_LOAD__) #define GC_API __declspec(dllexport) #else #define GC_API extern __declspec(dllimport) #endif #elif defined(_MSC_VER)||defined(__DMC__)||defined(__BORLANDC__)||defined(__CYGWIN__) #ifdef GC_BUILD #define GC_API extern __declspec(dllexport) #else #define GC_API __declspec(dllimport) #endif #elif defined(__WATCOMC__) #ifdef GC_BUILD #define GC_API extern __declspec(dllexport) #else #define GC_API extern __declspec(dllimport) #endif #elif defined(__SYMBIAN32__) #ifdef GC_BUILD #define GC_API extern EXPORT_C #else #define GC_API extern IMPORT_C #endif #elif defined(__GNUC__) #if defined(GC_BUILD)&&!defined(GC_NO_VISIBILITY)&&(GC_GNUC_PREREQ(4,0)||defined(GC_VISIBILITY_HIDDEN_SET)) #define GC_API extern __attribute__((__visibility__("default"))) #endif #endif #endif #ifndef GC_API #define GC_API extern #endif #ifndef GC_CALL #define GC_CALL #endif #ifndef GC_CALLBACK #define GC_CALLBACK GC_CALL #endif #ifndef GC_ATTR_MALLOC #ifdef GC_OOM_FUNC_RETURNS_ALIAS #define GC_ATTR_MALLOC #elif GC_GNUC_PREREQ(3,1) #define GC_ATTR_MALLOC __attribute__((__malloc__)) #elif defined(_MSC_VER)&&(_MSC_VER>=1900)&&!defined(__EDG__) #define GC_ATTR_MALLOC __declspec(allocator)__declspec(noalias)__declspec(restrict) #elif defined(_MSC_VER)&&_MSC_VER>=1400 #define GC_ATTR_MALLOC __declspec(noalias)__declspec(restrict) #else #define GC_ATTR_MALLOC #endif #endif #ifndef GC_ATTR_ALLOC_SIZE #undef GC_ATTR_CALLOC_SIZE #ifdef __clang__ #if __has_attribute(__alloc_size__) #define GC_ATTR_ALLOC_SIZE(argnum)__attribute__((__alloc_size__(argnum))) #define GC_ATTR_CALLOC_SIZE(n,s)__attribute__((__alloc_size__(n,s))) #else #define GC_ATTR_ALLOC_SIZE(argnum) #endif #elif GC_GNUC_PREREQ(4,3)&&!defined(__ICC) #define GC_ATTR_ALLOC_SIZE(argnum)__attribute__((__alloc_size__(argnum))) #define GC_ATTR_CALLOC_SIZE(n,s)__attribute__((__alloc_size__(n,s))) #else #define GC_ATTR_ALLOC_SIZE(argnum) #endif #endif #ifndef GC_ATTR_CALLOC_SIZE #define GC_ATTR_CALLOC_SIZE(n,s) #endif #ifndef GC_ATTR_NONNULL #if GC_GNUC_PREREQ(4,0) #define GC_ATTR_NONNULL(argnum)__attribute__((__nonnull__(argnum))) #else #define GC_ATTR_NONNULL(argnum) #endif #endif #ifndef GC_ATTR_CONST #if GC_GNUC_PREREQ(4,0) #define GC_ATTR_CONST __attribute__((__const__)) #else #define GC_ATTR_CONST #endif #endif #ifndef GC_ATTR_DEPRECATED #ifdef GC_BUILD #undef GC_ATTR_DEPRECATED #define GC_ATTR_DEPRECATED #elif GC_GNUC_PREREQ(4,0) #define GC_ATTR_DEPRECATED __attribute__((__deprecated__)) #elif defined(_MSC_VER)&&_MSC_VER>=1200 #define GC_ATTR_DEPRECATED __declspec(deprecated) #else #define GC_ATTR_DEPRECATED #endif #endif #if defined(__sgi)&&!defined(__GNUC__)&&_COMPILER_VERSION>=720 #define GC_ADD_CALLER #define GC_RETURN_ADDR (GC_word)__return_address #endif #if defined(__linux__)||defined(__GLIBC__) #if!defined(__native_client__) #include #endif #if (__GLIBC__==2&&__GLIBC_MINOR__>=1||__GLIBC__ > 2)&&!defined(__ia64__)&&!defined(GC_MISSING_EXECINFO_H)&&!defined(GC_HAVE_BUILTIN_BACKTRACE) #define GC_HAVE_BUILTIN_BACKTRACE #endif #if defined(__i386__)||defined(__amd64__)||defined(__x86_64__) #define GC_CAN_SAVE_CALL_STACKS #endif #endif #if defined(_MSC_VER)&&_MSC_VER>=1200&&!defined(_AMD64_)&&!defined(_M_X64)&&!defined(_WIN32_WCE)&&!defined(GC_HAVE_NO_BUILTIN_BACKTRACE)&&!defined(GC_HAVE_BUILTIN_BACKTRACE) #define GC_HAVE_BUILTIN_BACKTRACE #endif #if defined(GC_HAVE_BUILTIN_BACKTRACE)&&!defined(GC_CAN_SAVE_CALL_STACKS) #define GC_CAN_SAVE_CALL_STACKS #endif #if defined(__sparc__) #define GC_CAN_SAVE_CALL_STACKS #endif #if (defined(__linux__)||defined(__DragonFly__)||defined(__FreeBSD__)||defined(__FreeBSD_kernel__)||defined(__HAIKU__)||defined(__NetBSD__)||defined(__OpenBSD__)||defined(HOST_ANDROID)||defined(__ANDROID__))&&!defined(GC_CAN_SAVE_CALL_STACKS) #define GC_ADD_CALLER #if GC_GNUC_PREREQ(2,95) #define GC_RETURN_ADDR (GC_word)__builtin_return_address(0) #if GC_GNUC_PREREQ(4,0)&&(defined(__i386__)||defined(__amd64__)||defined(__x86_64__)) #define GC_HAVE_RETURN_ADDR_PARENT #define GC_RETURN_ADDR_PARENT (GC_word)__builtin_extract_return_addr(__builtin_return_address(1)) #endif #else #define GC_RETURN_ADDR 0 #endif #endif #ifdef GC_PTHREADS #if (defined(GC_DARWIN_THREADS)||defined(GC_WIN32_PTHREADS)||defined(__native_client__)||defined(GC_RTEMS_PTHREADS))&&!defined(GC_NO_DLOPEN) #define GC_NO_DLOPEN #endif #if (defined(GC_DARWIN_THREADS)||defined(GC_WIN32_PTHREADS)||defined(GC_OPENBSD_THREADS)||defined(__native_client__))&&!defined(GC_NO_PTHREAD_SIGMASK) #define GC_NO_PTHREAD_SIGMASK #endif #if defined(__native_client__) #ifndef GC_PTHREAD_CREATE_CONST #define GC_PTHREAD_CREATE_CONST #endif #ifndef GC_HAVE_PTHREAD_EXIT #define GC_HAVE_PTHREAD_EXIT #define GC_PTHREAD_EXIT_ATTRIBUTE #endif #endif #if!defined(GC_HAVE_PTHREAD_EXIT)&&!defined(HOST_ANDROID)&&!defined(__ANDROID__)&&(defined(GC_LINUX_THREADS)||defined(GC_SOLARIS_THREADS)) #define GC_HAVE_PTHREAD_EXIT #if GC_GNUC_PREREQ(2,7) #define GC_PTHREAD_EXIT_ATTRIBUTE __attribute__((__noreturn__)) #elif defined(__NORETURN) #define GC_PTHREAD_EXIT_ATTRIBUTE __NORETURN #else #define GC_PTHREAD_EXIT_ATTRIBUTE #endif #endif #if (!defined(GC_HAVE_PTHREAD_EXIT)||defined(__native_client__))&&!defined(GC_NO_PTHREAD_CANCEL) #define GC_NO_PTHREAD_CANCEL #endif #endif #ifdef __cplusplus #ifndef GC_ATTR_EXPLICIT #if __cplusplus>=201103L&&!defined(__clang__)||_MSVC_LANG>=201103L||defined(CPPCHECK) #define GC_ATTR_EXPLICIT explicit #else #define GC_ATTR_EXPLICIT #endif #endif #ifndef GC_NOEXCEPT #if defined(__DMC__)||(defined(__BORLANDC__)&&(defined(_RWSTD_NO_EXCEPTIONS)||defined(_RWSTD_NO_EX_SPEC)))||(defined(_MSC_VER)&&defined(_HAS_EXCEPTIONS)&&!_HAS_EXCEPTIONS)||(defined(__WATCOMC__)&&!defined(_CPPUNWIND)) #define GC_NOEXCEPT #ifndef GC_NEW_ABORTS_ON_OOM #define GC_NEW_ABORTS_ON_OOM #endif #elif __cplusplus>=201103L||_MSVC_LANG>=201103L #define GC_NOEXCEPT noexcept #else #define GC_NOEXCEPT throw() #endif #endif #endif #endif #ifdef __cplusplus extern "C" { #endif typedef void*GC_PTR; #ifdef _WIN64 #if defined(__int64)&&!defined(CPPCHECK) typedef unsigned __int64 GC_word; typedef __int64 GC_signed_word; #else typedef unsigned long long GC_word; typedef long long GC_signed_word; #endif #else typedef unsigned long GC_word; typedef long GC_signed_word; #endif GC_API unsigned GC_CALL GC_get_version(void); GC_API GC_ATTR_DEPRECATED GC_word GC_gc_no; GC_API GC_word GC_CALL GC_get_gc_no(void); #ifdef GC_THREADS GC_API GC_ATTR_DEPRECATED int GC_parallel; GC_API int GC_CALL GC_get_parallel(void); GC_API void GC_CALL GC_set_markers_count(unsigned); #endif typedef void*(GC_CALLBACK*GC_oom_func)(size_t); GC_API GC_ATTR_DEPRECATED GC_oom_func GC_oom_fn; GC_API void GC_CALL GC_set_oom_fn(GC_oom_func)GC_ATTR_NONNULL(1); GC_API GC_oom_func GC_CALL GC_get_oom_fn(void); typedef void (GC_CALLBACK*GC_on_heap_resize_proc)(GC_word); GC_API GC_ATTR_DEPRECATED GC_on_heap_resize_proc GC_on_heap_resize; GC_API void GC_CALL GC_set_on_heap_resize(GC_on_heap_resize_proc); GC_API GC_on_heap_resize_proc GC_CALL GC_get_on_heap_resize(void); typedef enum { GC_EVENT_START, GC_EVENT_MARK_START, GC_EVENT_MARK_END, GC_EVENT_RECLAIM_START, GC_EVENT_RECLAIM_END, GC_EVENT_END, GC_EVENT_PRE_STOP_WORLD, GC_EVENT_POST_STOP_WORLD, GC_EVENT_PRE_START_WORLD, GC_EVENT_POST_START_WORLD, GC_EVENT_THREAD_SUSPENDED, GC_EVENT_THREAD_UNSUSPENDED } GC_EventType; typedef void (GC_CALLBACK*GC_on_collection_event_proc)(GC_EventType); GC_API void GC_CALL GC_set_on_collection_event(GC_on_collection_event_proc); GC_API GC_on_collection_event_proc GC_CALL GC_get_on_collection_event(void); #if defined(GC_THREADS)||(defined(GC_BUILD)&&defined(NN_PLATFORM_CTR)) typedef void (GC_CALLBACK*GC_on_thread_event_proc)(GC_EventType, void*); GC_API void GC_CALL GC_set_on_thread_event(GC_on_thread_event_proc); GC_API GC_on_thread_event_proc GC_CALL GC_get_on_thread_event(void); #endif GC_API GC_ATTR_DEPRECATED int GC_find_leak; GC_API void GC_CALL GC_set_find_leak(int); GC_API int GC_CALL GC_get_find_leak(void); GC_API GC_ATTR_DEPRECATED int GC_all_interior_pointers; GC_API void GC_CALL GC_set_all_interior_pointers(int); GC_API int GC_CALL GC_get_all_interior_pointers(void); GC_API GC_ATTR_DEPRECATED int GC_finalize_on_demand; GC_API void GC_CALL GC_set_finalize_on_demand(int); GC_API int GC_CALL GC_get_finalize_on_demand(void); GC_API GC_ATTR_DEPRECATED int GC_java_finalization; GC_API void GC_CALL GC_set_java_finalization(int); GC_API int GC_CALL GC_get_java_finalization(void); typedef void (GC_CALLBACK*GC_finalizer_notifier_proc)(void); GC_API GC_ATTR_DEPRECATED GC_finalizer_notifier_proc GC_finalizer_notifier; GC_API void GC_CALL GC_set_finalizer_notifier(GC_finalizer_notifier_proc); GC_API GC_finalizer_notifier_proc GC_CALL GC_get_finalizer_notifier(void); GC_API #ifndef GC_DONT_GC GC_ATTR_DEPRECATED #endif int GC_dont_gc; GC_API GC_ATTR_DEPRECATED int GC_dont_expand; GC_API void GC_CALL GC_set_dont_expand(int); GC_API int GC_CALL GC_get_dont_expand(void); GC_API GC_ATTR_DEPRECATED int GC_use_entire_heap; GC_API GC_ATTR_DEPRECATED int GC_full_freq; GC_API void GC_CALL GC_set_full_freq(int); GC_API int GC_CALL GC_get_full_freq(void); GC_API GC_ATTR_DEPRECATED GC_word GC_non_gc_bytes; GC_API void GC_CALL GC_set_non_gc_bytes(GC_word); GC_API GC_word GC_CALL GC_get_non_gc_bytes(void); GC_API GC_ATTR_DEPRECATED int GC_no_dls; GC_API void GC_CALL GC_set_no_dls(int); GC_API int GC_CALL GC_get_no_dls(void); GC_API GC_ATTR_DEPRECATED GC_word GC_free_space_divisor; GC_API void GC_CALL GC_set_free_space_divisor(GC_word); GC_API GC_word GC_CALL GC_get_free_space_divisor(void); GC_API GC_ATTR_DEPRECATED GC_word GC_max_retries; GC_API void GC_CALL GC_set_max_retries(GC_word); GC_API GC_word GC_CALL GC_get_max_retries(void); GC_API GC_ATTR_DEPRECATED char*GC_stackbottom; GC_API GC_ATTR_DEPRECATED int GC_dont_precollect; GC_API void GC_CALL GC_set_dont_precollect(int); GC_API int GC_CALL GC_get_dont_precollect(void); GC_API GC_ATTR_DEPRECATED unsigned long GC_time_limit; #define GC_TIME_UNLIMITED 999999 GC_API void GC_CALL GC_set_time_limit(unsigned long); GC_API unsigned long GC_CALL GC_get_time_limit(void); struct GC_timeval_s { unsigned long tv_ms; unsigned long tv_nsec; }; GC_API void GC_CALL GC_set_time_limit_tv(struct GC_timeval_s); GC_API struct GC_timeval_s GC_CALL GC_get_time_limit_tv(void); GC_API void GC_CALL GC_set_allocd_bytes_per_finalizer(GC_word); GC_API GC_word GC_CALL GC_get_allocd_bytes_per_finalizer(void); GC_API void GC_CALL GC_start_performance_measurement(void); GC_API unsigned long GC_CALL GC_get_full_gc_total_time(void); GC_API void GC_CALL GC_set_pages_executable(int); GC_API int GC_CALL GC_get_pages_executable(void); GC_API void GC_CALL GC_set_min_bytes_allocd(size_t); GC_API size_t GC_CALL GC_get_min_bytes_allocd(void); GC_API void GC_CALL GC_set_rate(int); GC_API int GC_CALL GC_get_rate(void); GC_API void GC_CALL GC_set_max_prior_attempts(int); GC_API int GC_CALL GC_get_max_prior_attempts(void); GC_API void GC_CALL GC_set_handle_fork(int); GC_API void GC_CALL GC_atfork_prepare(void); GC_API void GC_CALL GC_atfork_parent(void); GC_API void GC_CALL GC_atfork_child(void); GC_API void GC_CALL GC_init(void); GC_API int GC_CALL GC_is_init_called(void); GC_API void GC_CALL GC_deinit(void); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_malloc(size_t); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_malloc_atomic(size_t); GC_API GC_ATTR_MALLOC char*GC_CALL GC_strdup(const char*); GC_API GC_ATTR_MALLOC char*GC_CALL GC_strndup(const char*,size_t)GC_ATTR_NONNULL(1); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_malloc_uncollectable(size_t); GC_API GC_ATTR_DEPRECATED void*GC_CALL GC_malloc_stubborn(size_t); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(2)void*GC_CALL GC_memalign(size_t,size_t); GC_API int GC_CALL GC_posix_memalign(void**,size_t, size_t)GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_free(void*); #define GC_MALLOC_STUBBORN(sz)GC_MALLOC(sz) #define GC_NEW_STUBBORN(t)GC_NEW(t) #define GC_CHANGE_STUBBORN(p)GC_change_stubborn(p) GC_API GC_ATTR_DEPRECATED void GC_CALL GC_change_stubborn(const void*); GC_API void GC_CALL GC_end_stubborn_change(const void*)GC_ATTR_NONNULL(1); GC_API void*GC_CALL GC_base(void*); GC_API int GC_CALL GC_is_heap_ptr(const void*); GC_API size_t GC_CALL GC_size(const void*)GC_ATTR_NONNULL(1); GC_API void*GC_CALL GC_realloc(void*, size_t) GC_ATTR_ALLOC_SIZE(2); GC_API int GC_CALL GC_expand_hp(size_t); GC_API void GC_CALL GC_set_max_heap_size(GC_word); GC_API void GC_CALL GC_exclude_static_roots(void*, void*); GC_API void GC_CALL GC_clear_exclusion_table(void); GC_API void GC_CALL GC_clear_roots(void); GC_API void GC_CALL GC_add_roots(void*, void*); GC_API void GC_CALL GC_remove_roots(void*, void*); GC_API void GC_CALL GC_register_displacement(size_t); GC_API void GC_CALL GC_debug_register_displacement(size_t); GC_API void GC_CALL GC_gcollect(void); GC_API void GC_CALL GC_gcollect_and_unmap(void); typedef int (GC_CALLBACK*GC_stop_func)(void); GC_API int GC_CALL GC_try_to_collect(GC_stop_func) GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_set_stop_func(GC_stop_func) GC_ATTR_NONNULL(1); GC_API GC_stop_func GC_CALL GC_get_stop_func(void); GC_API size_t GC_CALL GC_get_heap_size(void); GC_API size_t GC_CALL GC_get_free_bytes(void); GC_API size_t GC_CALL GC_get_unmapped_bytes(void); GC_API size_t GC_CALL GC_get_bytes_since_gc(void); GC_API size_t GC_CALL GC_get_expl_freed_bytes_since_gc(void); GC_API size_t GC_CALL GC_get_total_bytes(void); GC_API void GC_CALL GC_get_heap_usage_safe(GC_word*, GC_word*, GC_word*, GC_word*, GC_word*); struct GC_prof_stats_s { GC_word heapsize_full; GC_word free_bytes_full; GC_word unmapped_bytes; GC_word bytes_allocd_since_gc; GC_word allocd_bytes_before_gc; GC_word non_gc_bytes; GC_word gc_no; GC_word markers_m1; GC_word bytes_reclaimed_since_gc; GC_word reclaimed_bytes_before_gc; GC_word expl_freed_bytes_since_gc; }; GC_API size_t GC_CALL GC_get_prof_stats(struct GC_prof_stats_s*, size_t); #ifdef GC_THREADS GC_API size_t GC_CALL GC_get_prof_stats_unsafe(struct GC_prof_stats_s*, size_t); #endif GC_API size_t GC_CALL GC_get_size_map_at(int i); GC_API size_t GC_CALL GC_get_memory_use(void); GC_API void GC_CALL GC_disable(void); GC_API int GC_CALL GC_is_disabled(void); GC_API void GC_CALL GC_enable(void); GC_API void GC_CALL GC_set_manual_vdb_allowed(int); GC_API int GC_CALL GC_get_manual_vdb_allowed(void); GC_API void GC_CALL GC_enable_incremental(void); GC_API int GC_CALL GC_is_incremental_mode(void); #define GC_PROTECTS_POINTER_HEAP 1 #define GC_PROTECTS_PTRFREE_HEAP 2 #define GC_PROTECTS_STATIC_DATA 4 #define GC_PROTECTS_STACK 8 #define GC_PROTECTS_NONE 0 GC_API int GC_CALL GC_incremental_protection_needs(void); GC_API int GC_CALL GC_collect_a_little(void); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_malloc_ignore_off_page(size_t); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_malloc_atomic_ignore_off_page(size_t); #ifdef GC_ADD_CALLER #define GC_EXTRAS GC_RETURN_ADDR,__FILE__,__LINE__ #define GC_EXTRA_PARAMS GC_word ra,const char*s,int i #else #define GC_EXTRAS __FILE__,__LINE__ #define GC_EXTRA_PARAMS const char*s,int i #endif GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_malloc_atomic_uncollectable(size_t); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_debug_malloc_atomic_uncollectable(size_t,GC_EXTRA_PARAMS); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_debug_malloc(size_t,GC_EXTRA_PARAMS); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_debug_malloc_atomic(size_t,GC_EXTRA_PARAMS); GC_API GC_ATTR_MALLOC char*GC_CALL GC_debug_strdup(const char*,GC_EXTRA_PARAMS); GC_API GC_ATTR_MALLOC char*GC_CALL GC_debug_strndup(const char*,size_t,GC_EXTRA_PARAMS) GC_ATTR_NONNULL(1); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_debug_malloc_uncollectable(size_t, GC_EXTRA_PARAMS); GC_API GC_ATTR_DEPRECATED void*GC_CALL GC_debug_malloc_stubborn(size_t,GC_EXTRA_PARAMS); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_debug_malloc_ignore_off_page(size_t, GC_EXTRA_PARAMS); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_debug_malloc_atomic_ignore_off_page(size_t, GC_EXTRA_PARAMS); GC_API void GC_CALL GC_debug_free(void*); GC_API void*GC_CALL GC_debug_realloc(void*, size_t,GC_EXTRA_PARAMS) GC_ATTR_ALLOC_SIZE(2); GC_API GC_ATTR_DEPRECATED void GC_CALL GC_debug_change_stubborn(const void*); GC_API void GC_CALL GC_debug_end_stubborn_change(const void*) GC_ATTR_NONNULL(1); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_debug_malloc_replacement(size_t); GC_API GC_ATTR_ALLOC_SIZE(2)void*GC_CALL GC_debug_realloc_replacement(void*, size_t); #ifdef GC_DEBUG_REPLACEMENT #define GC_MALLOC(sz)GC_debug_malloc_replacement(sz) #define GC_REALLOC(old,sz)GC_debug_realloc_replacement(old,sz) #elif defined(GC_DEBUG) #define GC_MALLOC(sz)GC_debug_malloc(sz,GC_EXTRAS) #define GC_REALLOC(old,sz)GC_debug_realloc(old,sz,GC_EXTRAS) #else #define GC_MALLOC(sz)GC_malloc(sz) #define GC_REALLOC(old,sz)GC_realloc(old,sz) #endif #ifdef GC_DEBUG #define GC_MALLOC_ATOMIC(sz)GC_debug_malloc_atomic(sz,GC_EXTRAS) #define GC_STRDUP(s)GC_debug_strdup(s,GC_EXTRAS) #define GC_STRNDUP(s,sz)GC_debug_strndup(s,sz,GC_EXTRAS) #define GC_MALLOC_ATOMIC_UNCOLLECTABLE(sz)GC_debug_malloc_atomic_uncollectable(sz,GC_EXTRAS) #define GC_MALLOC_UNCOLLECTABLE(sz)GC_debug_malloc_uncollectable(sz,GC_EXTRAS) #define GC_MALLOC_IGNORE_OFF_PAGE(sz)GC_debug_malloc_ignore_off_page(sz,GC_EXTRAS) #define GC_MALLOC_ATOMIC_IGNORE_OFF_PAGE(sz)GC_debug_malloc_atomic_ignore_off_page(sz,GC_EXTRAS) #define GC_FREE(p)GC_debug_free(p) #define GC_REGISTER_FINALIZER(p,f,d,of,od)GC_debug_register_finalizer(p,f,d,of,od) #define GC_REGISTER_FINALIZER_IGNORE_SELF(p,f,d,of,od)GC_debug_register_finalizer_ignore_self(p,f,d,of,od) #define GC_REGISTER_FINALIZER_NO_ORDER(p,f,d,of,od)GC_debug_register_finalizer_no_order(p,f,d,of,od) #define GC_REGISTER_FINALIZER_UNREACHABLE(p,f,d,of,od)GC_debug_register_finalizer_unreachable(p,f,d,of,od) #define GC_END_STUBBORN_CHANGE(p)GC_debug_end_stubborn_change(p) #define GC_PTR_STORE_AND_DIRTY(p,q)GC_debug_ptr_store_and_dirty(p,q) #define GC_GENERAL_REGISTER_DISAPPEARING_LINK(link,obj)GC_general_register_disappearing_link(link,GC_base(( void*)(obj))) #define GC_REGISTER_LONG_LINK(link,obj)GC_register_long_link(link,GC_base(( void*)(obj))) #define GC_REGISTER_DISPLACEMENT(n)GC_debug_register_displacement(n) #else #define GC_MALLOC_ATOMIC(sz)GC_malloc_atomic(sz) #define GC_STRDUP(s)GC_strdup(s) #define GC_STRNDUP(s,sz)GC_strndup(s,sz) #define GC_MALLOC_ATOMIC_UNCOLLECTABLE(sz)GC_malloc_atomic_uncollectable(sz) #define GC_MALLOC_UNCOLLECTABLE(sz)GC_malloc_uncollectable(sz) #define GC_MALLOC_IGNORE_OFF_PAGE(sz)GC_malloc_ignore_off_page(sz) #define GC_MALLOC_ATOMIC_IGNORE_OFF_PAGE(sz)GC_malloc_atomic_ignore_off_page(sz) #define GC_FREE(p)GC_free(p) #define GC_REGISTER_FINALIZER(p,f,d,of,od)GC_register_finalizer(p,f,d,of,od) #define GC_REGISTER_FINALIZER_IGNORE_SELF(p,f,d,of,od)GC_register_finalizer_ignore_self(p,f,d,of,od) #define GC_REGISTER_FINALIZER_NO_ORDER(p,f,d,of,od)GC_register_finalizer_no_order(p,f,d,of,od) #define GC_REGISTER_FINALIZER_UNREACHABLE(p,f,d,of,od)GC_register_finalizer_unreachable(p,f,d,of,od) #define GC_END_STUBBORN_CHANGE(p)GC_end_stubborn_change(p) #define GC_PTR_STORE_AND_DIRTY(p,q)GC_ptr_store_and_dirty(p,q) #define GC_GENERAL_REGISTER_DISAPPEARING_LINK(link,obj)GC_general_register_disappearing_link(link,obj) #define GC_REGISTER_LONG_LINK(link,obj)GC_register_long_link(link,obj) #define GC_REGISTER_DISPLACEMENT(n)GC_register_displacement(n) #endif #define GC_NEW(t)((t*)GC_MALLOC(sizeof(t))) #define GC_NEW_ATOMIC(t)((t*)GC_MALLOC_ATOMIC(sizeof(t))) #define GC_NEW_UNCOLLECTABLE(t)((t*)GC_MALLOC_UNCOLLECTABLE(sizeof(t))) #ifdef GC_REQUIRE_WCSDUP GC_API GC_ATTR_MALLOC wchar_t*GC_CALL GC_wcsdup(const wchar_t*)GC_ATTR_NONNULL(1); GC_API GC_ATTR_MALLOC wchar_t*GC_CALL GC_debug_wcsdup(const wchar_t*,GC_EXTRA_PARAMS)GC_ATTR_NONNULL(1); #ifdef GC_DEBUG #define GC_WCSDUP(s)GC_debug_wcsdup(s,GC_EXTRAS) #else #define GC_WCSDUP(s)GC_wcsdup(s) #endif #endif typedef void (GC_CALLBACK*GC_finalization_proc)(void*, void*); GC_API void GC_CALL GC_register_finalizer(void*, GC_finalization_proc,void*, GC_finalization_proc*,void**) GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_debug_register_finalizer(void*, GC_finalization_proc,void*, GC_finalization_proc*,void**) GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_register_finalizer_ignore_self(void*, GC_finalization_proc,void*, GC_finalization_proc*,void**) GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_debug_register_finalizer_ignore_self(void*, GC_finalization_proc,void*, GC_finalization_proc*,void**) GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_register_finalizer_no_order(void*, GC_finalization_proc,void*, GC_finalization_proc*,void**) GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_debug_register_finalizer_no_order(void*, GC_finalization_proc,void*, GC_finalization_proc*,void**) GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_register_finalizer_unreachable(void*, GC_finalization_proc,void*, GC_finalization_proc*,void**) GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_debug_register_finalizer_unreachable(void*, GC_finalization_proc,void*, GC_finalization_proc*,void**) GC_ATTR_NONNULL(1); #define GC_NO_MEMORY 2 GC_API int GC_CALL GC_register_disappearing_link(void**) GC_ATTR_NONNULL(1); GC_API int GC_CALL GC_general_register_disappearing_link(void**, const void*) GC_ATTR_NONNULL(1)GC_ATTR_NONNULL(2); GC_API int GC_CALL GC_move_disappearing_link(void**, void**) GC_ATTR_NONNULL(2); GC_API int GC_CALL GC_unregister_disappearing_link(void**); GC_API int GC_CALL GC_register_long_link(void**, const void*) GC_ATTR_NONNULL(1)GC_ATTR_NONNULL(2); GC_API int GC_CALL GC_move_long_link(void**, void**) GC_ATTR_NONNULL(2); GC_API int GC_CALL GC_unregister_long_link(void**); typedef enum { GC_TOGGLE_REF_DROP, GC_TOGGLE_REF_STRONG, GC_TOGGLE_REF_WEAK } GC_ToggleRefStatus; typedef GC_ToggleRefStatus (GC_CALLBACK*GC_toggleref_func)(void*); GC_API void GC_CALL GC_set_toggleref_func(GC_toggleref_func); GC_API GC_toggleref_func GC_CALL GC_get_toggleref_func(void); GC_API int GC_CALL GC_toggleref_add(void*,int) GC_ATTR_NONNULL(1); typedef void (GC_CALLBACK*GC_await_finalize_proc)(void*); GC_API void GC_CALL GC_set_await_finalize_proc(GC_await_finalize_proc); GC_API GC_await_finalize_proc GC_CALL GC_get_await_finalize_proc(void); GC_API int GC_CALL GC_should_invoke_finalizers(void); GC_API int GC_CALL GC_invoke_finalizers(void); #if defined(__GNUC__)&&!defined(__INTEL_COMPILER) #define GC_reachable_here(ptr)__asm__ __volatile__(" "::"X"(ptr):"memory") #else GC_API void GC_CALL GC_noop1(GC_word); #ifdef LINT2 #define GC_reachable_here(ptr)GC_noop1(~(GC_word)(ptr)^(~(GC_word)0)) #else #define GC_reachable_here(ptr)GC_noop1((GC_word)(ptr)) #endif #endif typedef void (GC_CALLBACK*GC_warn_proc)(char*, GC_word); GC_API void GC_CALL GC_set_warn_proc(GC_warn_proc)GC_ATTR_NONNULL(1); GC_API GC_warn_proc GC_CALL GC_get_warn_proc(void); GC_API void GC_CALLBACK GC_ignore_warn_proc(char*,GC_word); GC_API void GC_CALL GC_set_log_fd(int); typedef void (GC_CALLBACK*GC_abort_func)(const char*); GC_API void GC_CALL GC_set_abort_func(GC_abort_func)GC_ATTR_NONNULL(1); GC_API GC_abort_func GC_CALL GC_get_abort_func(void); GC_API void GC_CALL GC_abort_on_oom(void); typedef GC_word GC_hidden_pointer; #define GC_HIDE_POINTER(p)(~(GC_hidden_pointer)(p)) #define GC_REVEAL_POINTER(p)((void*)GC_HIDE_POINTER(p)) #if defined(I_HIDE_POINTERS)||defined(GC_I_HIDE_POINTERS) #define HIDE_POINTER(p)GC_HIDE_POINTER(p) #define REVEAL_POINTER(p)GC_REVEAL_POINTER(p) #endif #ifdef GC_THREADS GC_API void GC_CALL GC_alloc_lock(void); GC_API void GC_CALL GC_alloc_unlock(void); #else #define GC_alloc_lock()(void)0 #define GC_alloc_unlock()(void)0 #endif typedef void*(GC_CALLBACK*GC_fn_type)(void*); GC_API void*GC_CALL GC_call_with_alloc_lock(GC_fn_type, void*)GC_ATTR_NONNULL(1); struct GC_stack_base { void*mem_base; #if defined(__ia64)||defined(__ia64__)||defined(_M_IA64) void*reg_base; #endif }; typedef void*(GC_CALLBACK*GC_stack_base_func)( struct GC_stack_base*,void*); GC_API void*GC_CALL GC_call_with_stack_base(GC_stack_base_func, void*)GC_ATTR_NONNULL(1); #define GC_SUCCESS 0 #define GC_DUPLICATE 1 #define GC_NO_THREADS 2 #define GC_UNIMPLEMENTED 3 #define GC_NOT_FOUND 4 #if defined(GC_DARWIN_THREADS)||defined(GC_WIN32_THREADS) GC_API void GC_CALL GC_use_threads_discovery(void); #endif #ifdef GC_THREADS GC_API void GC_CALL GC_set_suspend_signal(int); GC_API void GC_CALL GC_set_thr_restart_signal(int); GC_API int GC_CALL GC_get_suspend_signal(void); GC_API int GC_CALL GC_get_thr_restart_signal(void); GC_API void GC_CALL GC_start_mark_threads(void); GC_API void GC_CALL GC_allow_register_threads(void); GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base*) GC_ATTR_NONNULL(1); GC_API int GC_CALL GC_thread_is_registered(void); GC_API void GC_CALL GC_register_altstack(void*, GC_word, void*, GC_word); GC_API int GC_CALL GC_unregister_my_thread(void); GC_API void GC_CALL GC_stop_world_external(void); GC_API void GC_CALL GC_start_world_external(void); #endif GC_API void*GC_CALL GC_do_blocking(GC_fn_type, void*)GC_ATTR_NONNULL(1); GC_API void*GC_CALL GC_call_with_gc_active(GC_fn_type, void*)GC_ATTR_NONNULL(1); GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base*) GC_ATTR_NONNULL(1); GC_API void*GC_CALL GC_get_my_stackbottom(struct GC_stack_base*) GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_set_stackbottom(void*, const struct GC_stack_base*) GC_ATTR_NONNULL(2); GC_API void*GC_CALL GC_same_obj(void*,void*); GC_API void*GC_CALL GC_pre_incr(void**,ptrdiff_t) GC_ATTR_NONNULL(1); GC_API void*GC_CALL GC_post_incr(void**,ptrdiff_t) GC_ATTR_NONNULL(1); GC_API void*GC_CALL GC_is_visible(void*); GC_API void*GC_CALL GC_is_valid_displacement(void*); GC_API void GC_CALL GC_dump(void); GC_API void GC_CALL GC_dump_named(const char*); GC_API void GC_CALL GC_dump_regions(void); GC_API void GC_CALL GC_dump_finalization(void); #if defined(GC_DEBUG)&&defined(__GNUC__) #define GC_PTR_ADD3(x,n,type_of_result)((type_of_result)GC_same_obj((x)+(n),(x))) #define GC_PRE_INCR3(x,n,type_of_result)((type_of_result)GC_pre_incr((void**)(&(x)),(n)*sizeof(*x))) #define GC_POST_INCR3(x,n,type_of_result)((type_of_result)GC_post_incr((void**)(&(x)),(n)*sizeof(*x))) #define GC_PTR_ADD(x,n)GC_PTR_ADD3(x,n,__typeof__(x)) #define GC_PRE_INCR(x,n)GC_PRE_INCR3(x,n,__typeof__(x)) #define GC_POST_INCR(x)GC_POST_INCR3(x,1,__typeof__(x)) #define GC_POST_DECR(x)GC_POST_INCR3(x,-1,__typeof__(x)) #else #define GC_PTR_ADD(x,n)((x)+(n)) #define GC_PRE_INCR(x,n)((x)+=(n)) #define GC_POST_INCR(x)((x)++) #define GC_POST_DECR(x)((x)--) #endif #ifdef GC_DEBUG #define GC_PTR_STORE(p,q)(*(void**)GC_is_visible((void*)(p))=GC_is_valid_displacement((void*)(q))) #else #define GC_PTR_STORE(p,q)(*(void**)(p)=(void*)(q)) #endif GC_API void GC_CALL GC_ptr_store_and_dirty(void*, const void*); GC_API void GC_CALL GC_debug_ptr_store_and_dirty(void*, const void*); GC_API void (GC_CALLBACK*GC_same_obj_print_proc)(void*, void*); GC_API void (GC_CALLBACK*GC_is_valid_displacement_print_proc)(void*); GC_API void (GC_CALLBACK*GC_is_visible_print_proc)(void*); #ifdef GC_PTHREADS #ifdef __cplusplus } #endif #ifndef GC_PTHREAD_REDIRECTS_H #define GC_PTHREAD_REDIRECTS_H #if defined(GC_H)&&defined(GC_PTHREADS) #ifndef GC_PTHREAD_REDIRECTS_ONLY #include #ifndef GC_NO_DLOPEN #include #endif #ifndef GC_NO_PTHREAD_SIGMASK #include #endif #ifdef __cplusplus extern "C" { #endif #ifndef GC_SUSPEND_THREAD_ID #define GC_SUSPEND_THREAD_ID pthread_t #endif #ifndef GC_NO_DLOPEN GC_API void*GC_dlopen(const char*,int); #endif #ifndef GC_NO_PTHREAD_SIGMASK #if defined(GC_PTHREAD_SIGMASK_NEEDED)||defined(_BSD_SOURCE)||defined(_GNU_SOURCE)||(_POSIX_C_SOURCE>=199506L)||(_XOPEN_SOURCE>=500) GC_API int GC_pthread_sigmask(int,const sigset_t*, sigset_t*); #endif #endif #ifndef GC_PTHREAD_CREATE_CONST #define GC_PTHREAD_CREATE_CONST const #endif GC_API int GC_pthread_create(pthread_t*, GC_PTHREAD_CREATE_CONST pthread_attr_t*, void*(*)(void*),void*); GC_API int GC_pthread_join(pthread_t,void**); GC_API int GC_pthread_detach(pthread_t); #ifndef GC_NO_PTHREAD_CANCEL GC_API int GC_pthread_cancel(pthread_t); #endif #if defined(GC_HAVE_PTHREAD_EXIT)&&!defined(GC_PTHREAD_EXIT_DECLARED) #define GC_PTHREAD_EXIT_DECLARED GC_API void GC_pthread_exit(void*)GC_PTHREAD_EXIT_ATTRIBUTE; #endif #ifdef __cplusplus } #endif #endif #if!defined(GC_NO_THREAD_REDIRECTS)&&!defined(GC_USE_LD_WRAP) #undef pthread_create #undef pthread_join #undef pthread_detach #define pthread_create GC_pthread_create #define pthread_join GC_pthread_join #define pthread_detach GC_pthread_detach #ifndef GC_NO_PTHREAD_SIGMASK #undef pthread_sigmask #define pthread_sigmask GC_pthread_sigmask #endif #ifndef GC_NO_DLOPEN #undef dlopen #define dlopen GC_dlopen #endif #ifndef GC_NO_PTHREAD_CANCEL #undef pthread_cancel #define pthread_cancel GC_pthread_cancel #endif #ifdef GC_HAVE_PTHREAD_EXIT #undef pthread_exit #define pthread_exit GC_pthread_exit #endif #endif #endif #endif #ifdef __cplusplus extern "C" { #endif #endif GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_many(size_t); #define GC_NEXT(p)(*(void**)(p)) typedef int (GC_CALLBACK*GC_has_static_roots_func)( const char*, void*, size_t); GC_API void GC_CALL GC_register_has_static_roots_callback( GC_has_static_roots_func); #if!defined(CPPCHECK)&&!defined(GC_WINDOWS_H_INCLUDED)&&defined(WINAPI) #define GC_WINDOWS_H_INCLUDED #endif #if defined(GC_WIN32_THREADS)&&(!defined(GC_PTHREADS)||defined(GC_BUILD)||defined(GC_WINDOWS_H_INCLUDED)) #if (!defined(GC_NO_THREAD_DECLS)||defined(GC_BUILD))&&!defined(GC_DONT_INCL_WINDOWS_H) #ifdef __cplusplus } #endif #if!defined(_WIN32_WCE)&&!defined(__CEGCC__) #include #endif #if defined(GC_BUILD)||!defined(GC_DONT_INCLUDE_WINDOWS_H) #include #define GC_WINDOWS_H_INCLUDED #endif #ifdef __cplusplus extern "C" { #endif #ifdef GC_UNDERSCORE_STDCALL #define GC_CreateThread _GC_CreateThread #define GC_ExitThread _GC_ExitThread #endif #ifndef DECLSPEC_NORETURN #ifdef GC_WINDOWS_H_INCLUDED #define DECLSPEC_NORETURN #else #define DECLSPEC_NORETURN __declspec(noreturn) #endif #endif #if!defined(_UINTPTR_T)&&!defined(_UINTPTR_T_DEFINED)&&!defined(UINTPTR_MAX) typedef GC_word GC_uintptr_t; #else typedef uintptr_t GC_uintptr_t; #endif #ifdef _WIN64 #define GC_WIN32_SIZE_T GC_uintptr_t #elif defined(GC_WINDOWS_H_INCLUDED) #define GC_WIN32_SIZE_T DWORD #else #define GC_WIN32_SIZE_T unsigned long #endif #ifdef GC_INSIDE_DLL #ifdef GC_UNDERSCORE_STDCALL #define GC_DllMain _GC_DllMain #endif #ifdef GC_WINDOWS_H_INCLUDED GC_API BOOL WINAPI GC_DllMain(HINSTANCE, ULONG, LPVOID); #else GC_API int __stdcall GC_DllMain(void*,unsigned long,void*); #endif #endif #ifdef GC_WINDOWS_H_INCLUDED GC_API HANDLE WINAPI GC_CreateThread( LPSECURITY_ATTRIBUTES, GC_WIN32_SIZE_T, LPTHREAD_START_ROUTINE, LPVOID,DWORD, LPDWORD); GC_API DECLSPEC_NORETURN void WINAPI GC_ExitThread( DWORD); #else struct _SECURITY_ATTRIBUTES; GC_API void*__stdcall GC_CreateThread(struct _SECURITY_ATTRIBUTES*, GC_WIN32_SIZE_T, unsigned long (__stdcall*)(void*), void*,unsigned long,unsigned long*); GC_API DECLSPEC_NORETURN void __stdcall GC_ExitThread(unsigned long); #endif #if!defined(_WIN32_WCE)&&!defined(__CEGCC__) GC_API GC_uintptr_t GC_CALL GC_beginthreadex( void*,unsigned, unsigned (__stdcall*)(void*), void*,unsigned, unsigned*); GC_API void GC_CALL GC_endthreadex(unsigned); #endif #endif #ifdef GC_WINMAIN_REDIRECT #define WinMain GC_WinMain #endif #define GC_use_DllMain GC_use_threads_discovery #ifndef GC_NO_THREAD_REDIRECTS #define CreateThread GC_CreateThread #define ExitThread GC_ExitThread #undef _beginthreadex #define _beginthreadex GC_beginthreadex #undef _endthreadex #define _endthreadex GC_endthreadex #endif #endif GC_API void GC_CALL GC_set_force_unmap_on_gcollect(int); GC_API int GC_CALL GC_get_force_unmap_on_gcollect(void); #if defined(__CYGWIN32__)||defined(__CYGWIN__) #ifdef __x86_64__ extern int __data_start__[],__data_end__[]; extern int __bss_start__[],__bss_end__[]; #define GC_DATASTART ((GC_word)__data_start__ < (GC_word)__bss_start__?(void*)__data_start__:(void*)__bss_start__) #define GC_DATAEND ((GC_word)__data_end__ > (GC_word)__bss_end__?(void*)__data_end__:(void*)__bss_end__) #else extern int _data_start__[],_data_end__[],_bss_start__[],_bss_end__[]; #define GC_DATASTART ((GC_word)_data_start__ < (GC_word)_bss_start__?(void*)_data_start__:(void*)_bss_start__) #define GC_DATAEND ((GC_word)_data_end__ > (GC_word)_bss_end__?(void*)_data_end__:(void*)_bss_end__) #endif #define GC_INIT_CONF_ROOTS GC_add_roots(GC_DATASTART,GC_DATAEND);GC_gcollect() #elif defined(_AIX) extern int _data[],_end[]; #define GC_DATASTART ((void*)_data) #define GC_DATAEND ((void*)_end) #define GC_INIT_CONF_ROOTS GC_add_roots(GC_DATASTART,GC_DATAEND) #elif (defined(HOST_ANDROID)||defined(__ANDROID__))&&defined(IGNORE_DYNAMIC_LOADING) #pragma weak __dso_handle extern int __dso_handle[]; GC_API void*GC_CALL GC_find_limit(void*,int); #define GC_INIT_CONF_ROOTS (void)(__dso_handle!=0?(GC_add_roots(__dso_handle,GC_find_limit(__dso_handle,1)),0):0) #else #define GC_INIT_CONF_ROOTS #endif #ifdef GC_DONT_EXPAND #define GC_INIT_CONF_DONT_EXPAND GC_set_dont_expand(1) #else #define GC_INIT_CONF_DONT_EXPAND #endif #ifdef GC_FORCE_UNMAP_ON_GCOLLECT #define GC_INIT_CONF_FORCE_UNMAP_ON_GCOLLECT GC_set_force_unmap_on_gcollect(1) #else #define GC_INIT_CONF_FORCE_UNMAP_ON_GCOLLECT #endif #ifdef GC_DONT_GC #define GC_INIT_CONF_MAX_RETRIES (void)(GC_dont_gc=1) #elif defined(GC_MAX_RETRIES)&&!defined(CPPCHECK) #define GC_INIT_CONF_MAX_RETRIES GC_set_max_retries(GC_MAX_RETRIES) #else #define GC_INIT_CONF_MAX_RETRIES #endif #if defined(GC_ALLOCD_BYTES_PER_FINALIZER)&&!defined(CPPCHECK) #define GC_INIT_CONF_ALLOCD_BYTES_PER_FINALIZER GC_set_allocd_bytes_per_finalizer(GC_ALLOCD_BYTES_PER_FINALIZER) #else #define GC_INIT_CONF_ALLOCD_BYTES_PER_FINALIZER #endif #if defined(GC_FREE_SPACE_DIVISOR)&&!defined(CPPCHECK) #define GC_INIT_CONF_FREE_SPACE_DIVISOR GC_set_free_space_divisor(GC_FREE_SPACE_DIVISOR) #else #define GC_INIT_CONF_FREE_SPACE_DIVISOR #endif #if defined(GC_FULL_FREQ)&&!defined(CPPCHECK) #define GC_INIT_CONF_FULL_FREQ GC_set_full_freq(GC_FULL_FREQ) #else #define GC_INIT_CONF_FULL_FREQ #endif #if defined(GC_TIME_LIMIT)&&!defined(CPPCHECK) #define GC_INIT_CONF_TIME_LIMIT GC_set_time_limit(GC_TIME_LIMIT) #else #define GC_INIT_CONF_TIME_LIMIT #endif #if defined(GC_MARKERS)&&defined(GC_THREADS)&&!defined(CPPCHECK) #define GC_INIT_CONF_MARKERS GC_set_markers_count(GC_MARKERS) #else #define GC_INIT_CONF_MARKERS #endif #if defined(GC_SIG_SUSPEND)&&defined(GC_THREADS)&&!defined(CPPCHECK) #define GC_INIT_CONF_SUSPEND_SIGNAL GC_set_suspend_signal(GC_SIG_SUSPEND) #else #define GC_INIT_CONF_SUSPEND_SIGNAL #endif #if defined(GC_SIG_THR_RESTART)&&defined(GC_THREADS)&&!defined(CPPCHECK) #define GC_INIT_CONF_THR_RESTART_SIGNAL GC_set_thr_restart_signal(GC_SIG_THR_RESTART) #else #define GC_INIT_CONF_THR_RESTART_SIGNAL #endif #if defined(GC_MAXIMUM_HEAP_SIZE)&&!defined(CPPCHECK) #define GC_INIT_CONF_MAXIMUM_HEAP_SIZE GC_set_max_heap_size(GC_MAXIMUM_HEAP_SIZE) #else #define GC_INIT_CONF_MAXIMUM_HEAP_SIZE #endif #ifdef GC_IGNORE_WARN #define GC_INIT_CONF_IGNORE_WARN GC_set_warn_proc(GC_ignore_warn_proc) #else #define GC_INIT_CONF_IGNORE_WARN #endif #if defined(GC_INITIAL_HEAP_SIZE)&&!defined(CPPCHECK) #define GC_INIT_CONF_INITIAL_HEAP_SIZE { size_t heap_size=GC_get_heap_size();if (heap_size < (GC_INITIAL_HEAP_SIZE))(void)GC_expand_hp((GC_INITIAL_HEAP_SIZE)- heap_size);} #else #define GC_INIT_CONF_INITIAL_HEAP_SIZE #endif #define GC_INIT(){ GC_INIT_CONF_DONT_EXPAND;GC_INIT_CONF_FORCE_UNMAP_ON_GCOLLECT;GC_INIT_CONF_MAX_RETRIES;GC_INIT_CONF_ALLOCD_BYTES_PER_FINALIZER;GC_INIT_CONF_FREE_SPACE_DIVISOR;GC_INIT_CONF_FULL_FREQ;GC_INIT_CONF_TIME_LIMIT;GC_INIT_CONF_MARKERS;GC_INIT_CONF_SUSPEND_SIGNAL;GC_INIT_CONF_THR_RESTART_SIGNAL;GC_INIT_CONF_MAXIMUM_HEAP_SIZE;GC_init();GC_INIT_CONF_ROOTS;GC_INIT_CONF_IGNORE_WARN;GC_INIT_CONF_INITIAL_HEAP_SIZE;} GC_API void GC_CALL GC_win32_free_heap(void); #if defined(__SYMBIAN32__) void GC_init_global_static_roots(void); #endif #if defined(_AMIGA)&&!defined(GC_AMIGA_MAKINGLIB) void*GC_amiga_realloc(void*,size_t); #define GC_realloc(a,b)GC_amiga_realloc(a,b) void GC_amiga_set_toany(void (*)(void)); extern int GC_amiga_free_space_divisor_inc; extern void*(*GC_amiga_allocwrapper_do)(size_t,void*(GC_CALL*)(size_t)); #define GC_malloc(a)(*GC_amiga_allocwrapper_do)(a,GC_malloc) #define GC_malloc_atomic(a)(*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic) #define GC_malloc_uncollectable(a)(*GC_amiga_allocwrapper_do)(a,GC_malloc_uncollectable) #define GC_malloc_atomic_uncollectable(a)(*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_uncollectable) #define GC_malloc_ignore_off_page(a)(*GC_amiga_allocwrapper_do)(a,GC_malloc_ignore_off_page) #define GC_malloc_atomic_ignore_off_page(a)(*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page) #endif #ifdef __cplusplus } #endif #endif #endif #include #if!defined(sony_news) #include #endif #ifdef DGUX #include #include #include #endif #ifdef BSD_TIME #include #include #include #endif #ifdef PARALLEL_MARK #define AO_REQUIRE_CAS #if!defined(__GNUC__)&&!defined(AO_ASSUME_WINDOWS98) #define AO_ASSUME_WINDOWS98 #endif #endif #ifndef GC_TINY_FL_H #define GC_TINY_FL_H #ifndef GC_GRANULE_BYTES #if defined(__LP64__)||defined (_LP64)||defined(_WIN64)||defined(__s390x__)||(defined(__x86_64__)&&!defined(__ILP32__))||defined(__alpha__)||defined(__powerpc64__)||defined(__arch64__) #define GC_GRANULE_BYTES 16 #define GC_GRANULE_WORDS 2 #else #define GC_GRANULE_BYTES 8 #define GC_GRANULE_WORDS 2 #endif #endif #if GC_GRANULE_WORDS==2 #define GC_WORDS_TO_GRANULES(n)((n)>>1) #else #define GC_WORDS_TO_GRANULES(n)((n)*sizeof(void*)/GC_GRANULE_BYTES) #endif #ifndef GC_TINY_FREELISTS #if GC_GRANULE_BYTES==16 #define GC_TINY_FREELISTS 25 #else #define GC_TINY_FREELISTS 33 #endif #endif #define GC_RAW_BYTES_FROM_INDEX(i)((i)*GC_GRANULE_BYTES) #endif #ifndef GC_MARK_H #define GC_MARK_H #ifndef GC_H #endif #ifdef __cplusplus extern "C" { #endif #define GC_PROC_BYTES 100 #if defined(GC_BUILD)||defined(NOT_GCBUILD) struct GC_ms_entry; #else struct GC_ms_entry { void*opaque;}; #endif typedef struct GC_ms_entry*(*GC_mark_proc)(GC_word*, struct GC_ms_entry*, struct GC_ms_entry*, GC_word); #define GC_LOG_MAX_MARK_PROCS 6 #define GC_MAX_MARK_PROCS (1<=(GC_word)GC_least_plausible_heap_addr&&(GC_word)(obj)<=(GC_word)GC_greatest_plausible_heap_addr?GC_mark_and_push(obj,msp,lim,src):(msp)) GC_API GC_ATTR_CONST size_t GC_CALL GC_get_debug_header_size(void); #define GC_USR_PTR_FROM_BASE(p)((void*)((char*)(p)+GC_get_debug_header_size())) GC_API GC_ATTR_DEPRECATED #ifdef GC_BUILD const #endif size_t GC_debug_header_size; GC_API void**GC_CALL GC_new_free_list(void); GC_API void**GC_CALL GC_new_free_list_inner(void); GC_API unsigned GC_CALL GC_new_kind(void**, GC_word, int, int)GC_ATTR_NONNULL(1); GC_API unsigned GC_CALL GC_new_kind_inner(void**, GC_word, int, int)GC_ATTR_NONNULL(1); GC_API unsigned GC_CALL GC_new_proc(GC_mark_proc); GC_API unsigned GC_CALL GC_new_proc_inner(GC_mark_proc); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_generic_malloc( size_t, int); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_generic_malloc_ignore_off_page( size_t,int); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_generic_malloc_uncollectable( size_t,int); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_generic_or_special_malloc( size_t,int); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_debug_generic_or_special_malloc( size_t,int, GC_EXTRA_PARAMS); #ifdef GC_DEBUG #define GC_GENERIC_OR_SPECIAL_MALLOC(sz,knd)GC_debug_generic_or_special_malloc(sz,knd,GC_EXTRAS) #else #define GC_GENERIC_OR_SPECIAL_MALLOC(sz,knd)GC_generic_or_special_malloc(sz,knd) #endif GC_API int GC_CALL GC_get_kind_and_size(const void*,size_t*) GC_ATTR_NONNULL(1); typedef void (GC_CALLBACK*GC_describe_type_fn)(void*, char*); #define GC_TYPE_DESCR_LEN 40 GC_API void GC_CALL GC_register_describe_type_fn(int, GC_describe_type_fn); GC_API void*GC_CALL GC_clear_stack(void*); typedef void (GC_CALLBACK*GC_start_callback_proc)(void); GC_API void GC_CALL GC_set_start_callback(GC_start_callback_proc); GC_API GC_start_callback_proc GC_CALL GC_get_start_callback(void); GC_API int GC_CALL GC_is_marked(const void*)GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_clear_mark_bit(const void*)GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_set_mark_bit(const void*)GC_ATTR_NONNULL(1); GC_API void GC_CALL GC_push_all(void*,void*); GC_API void GC_CALL GC_push_all_eager(void*,void*); GC_API void GC_CALL GC_push_conditional(void*,void*, int); GC_API void GC_CALL GC_push_finalizer_structures(void); typedef void (GC_CALLBACK*GC_push_other_roots_proc)(void); GC_API void GC_CALL GC_set_push_other_roots(GC_push_other_roots_proc); GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void); typedef void (GC_CALLBACK*GC_reachable_object_proc)(void*, size_t, void*); GC_API void GC_CALL GC_enumerate_reachable_objects_inner( GC_reachable_object_proc, void*)GC_ATTR_NONNULL(1); GC_API int GC_CALL GC_is_tmp_root(void*); GC_API void GC_CALL GC_print_trace(GC_word); GC_API void GC_CALL GC_print_trace_inner(GC_word); #ifdef __cplusplus } #endif #endif typedef GC_word word; typedef GC_signed_word signed_word; typedef unsigned int unsigned32; typedef int GC_bool; #define TRUE 1 #define FALSE 0 #ifndef PTR_T_DEFINED typedef char*ptr_t; #define PTR_T_DEFINED #endif #ifndef SIZE_MAX #include #endif #if defined(SIZE_MAX)&&!defined(CPPCHECK) #define GC_SIZE_MAX ((size_t)SIZE_MAX) #else #define GC_SIZE_MAX (~(size_t)0) #endif #if GC_GNUC_PREREQ(3,0)&&!defined(LINT2) #define EXPECT(expr,outcome)__builtin_expect(expr,outcome) #else #define EXPECT(expr,outcome)(expr) #endif #define SIZET_SAT_ADD(a,b)(EXPECT((a)< GC_SIZE_MAX - (b),TRUE)?(a)+(b):GC_SIZE_MAX) #ifndef GCCONFIG_H #define GCCONFIG_H #ifdef CPPCHECK #undef CLOCKS_PER_SEC #undef FIXUP_POINTER #undef POINTER_MASK #undef POINTER_SHIFT #undef REDIRECT_REALLOC #undef _MAX_PATH #endif #ifndef PTR_T_DEFINED typedef char*ptr_t; #define PTR_T_DEFINED #endif #if!defined(sony_news) #include #endif #ifdef __cplusplus #define EXTERN_C_BEGIN extern "C" { #define EXTERN_C_END } #else #define EXTERN_C_BEGIN #define EXTERN_C_END #endif EXTERN_C_BEGIN #if defined(__clang__)&&defined(__clang_major__) #define GC_CLANG_PREREQ(major,minor)((__clang_major__<<16)+__clang_minor__>=((major)<<16)+(minor)) #define GC_CLANG_PREREQ_FULL(major,minor,patchlevel)(GC_CLANG_PREREQ(major,(minor)+1)||(__clang_major__==(major)&&__clang_minor__==(minor)&&__clang_patchlevel__>=(patchlevel))) #else #define GC_CLANG_PREREQ(major,minor)0 #define GC_CLANG_PREREQ_FULL(major,minor,patchlevel)0 #endif #ifdef LINT2 #define COVERT_DATAFLOW(w)(~(GC_word)(w)^(~(GC_word)0)) #else #define COVERT_DATAFLOW(w)((GC_word)(w)) #endif #if defined(__ANDROID__)&&!defined(HOST_ANDROID) #define HOST_ANDROID 1 #endif #if defined(TIZEN)&&!defined(HOST_TIZEN) #define HOST_TIZEN 1 #endif #if defined(__SYMBIAN32__)&&!defined(SYMBIAN) #define SYMBIAN #ifdef __WINS__ #pragma data_seg(".data2") #endif #endif #if (defined(linux)||defined(__linux__)||defined(HOST_ANDROID))&&!defined(LINUX)&&!defined(__native_client__) #define LINUX #endif #if defined(__NetBSD__) #define NETBSD #endif #if defined(__OpenBSD__) #define OPENBSD #endif #if (defined(__FreeBSD__)||defined(__DragonFly__)||defined(__FreeBSD_kernel__))&&!defined(FREEBSD)&&!defined(SN_TARGET_ORBIS) #define FREEBSD #endif #if defined(macosx)||(defined(__APPLE__)&&defined(__MACH__)) #define DARWIN EXTERN_C_END #include EXTERN_C_BEGIN #endif #if defined(__native_client__) #define NACL #if!defined(__portable_native_client__)&&!defined(__arm__) #define I386 #define mach_type_known #else #endif #endif #if defined(__aarch64__) #define AARCH64 #if!defined(LINUX)&&!defined(DARWIN)&&!defined(FREEBSD)&&!defined(NETBSD)&&!defined(NN_BUILD_TARGET_PLATFORM_NX)&&!defined(OPENBSD) #define NOSYS #define mach_type_known #endif #endif #if defined(__arm)||defined(__arm__)||defined(__thumb__) #define ARM32 #if defined(NACL) #define mach_type_known #elif!defined(LINUX)&&!defined(NETBSD)&&!defined(FREEBSD)&&!defined(OPENBSD)&&!defined(DARWIN)&&!defined(_WIN32)&&!defined(__CEGCC__)&&!defined(NN_PLATFORM_CTR)&&!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PSP2)&&!defined(SYMBIAN) #define NOSYS #define mach_type_known #endif #endif #if defined(sun)&&defined(mc68000)&&!defined(CPPCHECK) #error SUNOS4 no longer supported #endif #if defined(hp9000s300)&&!defined(CPPCHECK) #error M68K based HP machines no longer supported #endif #if defined(OPENBSD)&&defined(m68k) #define M68K #define mach_type_known #endif #if defined(OPENBSD)&&defined(__sparc__) #define SPARC #define mach_type_known #endif #if defined(OPENBSD)&&defined(__arm__) #define ARM32 #define mach_type_known #endif #if defined(OPENBSD)&&defined(__aarch64__) #define AARCH64 #define mach_type_known #endif #if defined(OPENBSD)&&defined(__sh__) #define SH #define mach_type_known #endif #if defined(NETBSD)&&(defined(m68k)||defined(__m68k__)) #define M68K #define mach_type_known #endif #if defined(NETBSD)&&defined(__powerpc__) #define POWERPC #define mach_type_known #endif #if defined(NETBSD)&&(defined(__arm32__)||defined(__arm__)) #define ARM32 #define mach_type_known #endif #if defined(NETBSD)&&defined(__aarch64__) #define AARCH64 #define mach_type_known #endif #if defined(NETBSD)&&defined(__sh__) #define SH #define mach_type_known #endif #if defined(vax)||defined(__vax__) #define VAX #ifdef ultrix #define ULTRIX #else #define BSD #endif #define mach_type_known #endif #if defined(NETBSD)&&defined(__vax__) #define VAX #define mach_type_known #endif #if defined(mips)||defined(__mips)||defined(_mips) #define MIPS #if defined(nec_ews)||defined(_nec_ews) #define EWS4800 #endif #if!defined(LINUX)&&!defined(EWS4800)&&!defined(NETBSD)&&!defined(OPENBSD) #if defined(ultrix)||defined(__ultrix) #define ULTRIX #else #define IRIX5 #endif #endif #if defined(NETBSD)&&defined(__MIPSEL__) #undef ULTRIX #endif #define mach_type_known #endif #if defined(__QNX__) #define I386 #define mach_type_known #endif #if defined(__NIOS2__)||defined(__NIOS2)||defined(__nios2__) #define NIOS2 #define mach_type_known #endif #if defined(__or1k__) #define OR1K #define mach_type_known #endif #if defined(DGUX)&&(defined(i386)||defined(__i386__)) #define I386 #ifndef _USING_DGUX #define _USING_DGUX #endif #define mach_type_known #endif #if defined(sequent)&&(defined(i386)||defined(__i386__)) #define I386 #define SEQUENT #define mach_type_known #endif #if (defined(sun)||defined(__sun))&&(defined(i386)||defined(__i386__)) #define I386 #define SOLARIS #define mach_type_known #endif #if (defined(sun)||defined(__sun))&&defined(__amd64) #define X86_64 #define SOLARIS #define mach_type_known #endif #if (defined(__OS2__)||defined(__EMX__))&&defined(__32BIT__) #define I386 #define OS2 #define mach_type_known #endif #if defined(ibm032)&&!defined(CPPCHECK) #error IBM PC/RT no longer supported #endif #if (defined(sun)||defined(__sun))&&(defined(sparc)||defined(__sparc)) EXTERN_C_END #include EXTERN_C_BEGIN #define SPARC #define SOLARIS #define mach_type_known #elif defined(sparc)&&defined(unix)&&!defined(sun)&&!defined(linux)&&!defined(FREEBSD)&&!defined(NETBSD)&&!defined(OPENBSD) #define SPARC #define DRSNX #define mach_type_known #endif #if defined(_IBMR2) #define POWERPC #define AIX #define mach_type_known #endif #if defined(NETBSD)&&defined(__sparc__) #define SPARC #define mach_type_known #endif #if defined(_M_XENIX)&&defined(_M_SYSV)&&defined(_M_I386) #define I386 #if defined(_SCO_ELF) #define SCO_ELF #else #define SCO #endif #define mach_type_known #endif #if defined(_AUX_SOURCE)&&!defined(CPPCHECK) #error A/UX no longer supported #endif #if defined(_PA_RISC1_0)||defined(_PA_RISC1_1)||defined(_PA_RISC2_0)||defined(hppa)||defined(__hppa__) #define HP_PA #if!defined(LINUX)&&!defined(HPUX)&&!defined(OPENBSD) #define HPUX #endif #define mach_type_known #endif #if defined(__ia64)&&(defined(_HPUX_SOURCE)||defined(__HP_aCC)) #define IA64 #ifndef HPUX #define HPUX #endif #define mach_type_known #endif #if (defined(__BEOS__)||defined(__HAIKU__))&&defined(_X86_) #define I386 #define HAIKU #define mach_type_known #endif #if defined(__HAIKU__)&&(defined(__amd64__)||defined(__x86_64__)) #define X86_64 #define HAIKU #define mach_type_known #endif #if defined(OPENBSD)&&defined(__amd64__) #define X86_64 #define mach_type_known #endif #if defined(LINUX)&&(defined(i386)||defined(__i386__)) #define I386 #define mach_type_known #endif #if defined(LINUX)&&defined(__x86_64__) #define X86_64 #define mach_type_known #endif #if defined(LINUX)&&(defined(__ia64__)||defined(__ia64)) #define IA64 #define mach_type_known #endif #if defined(LINUX)&&defined(__aarch64__) #define AARCH64 #define mach_type_known #endif #if defined(LINUX)&&(defined(__arm)||defined(__arm__)) #define ARM32 #define mach_type_known #endif #if defined(LINUX)&&defined(__cris__) #ifndef CRIS #define CRIS #endif #define mach_type_known #endif #if defined(LINUX)&&(defined(powerpc)||defined(__powerpc__)||defined(powerpc64)||defined(__powerpc64__)) #define POWERPC #define mach_type_known #endif #if defined(LINUX)&&defined(__mc68000__) #define M68K #define mach_type_known #endif #if defined(LINUX)&&(defined(sparc)||defined(__sparc__)) #define SPARC #define mach_type_known #endif #if defined(LINUX)&&defined(__sh__) #define SH #define mach_type_known #endif #if defined(LINUX)&&defined(__avr32__) #define AVR32 #define mach_type_known #endif #if defined(LINUX)&&defined(__m32r__) #define M32R #define mach_type_known #endif #if defined(__alpha)||defined(__alpha__) #define ALPHA #if!defined(LINUX)&&!defined(NETBSD)&&!defined(OPENBSD)&&!defined(FREEBSD) #define OSF1 #endif #define mach_type_known #endif #if defined(_AMIGA)&&!defined(AMIGA) #define AMIGA #endif #ifdef AMIGA #define M68K #define mach_type_known #endif #if defined(THINK_C)||(defined(__MWERKS__)&&!defined(__powerc)&&!defined(SYMBIAN)) #define M68K #define MACOS #define mach_type_known #endif #if defined(__MWERKS__)&&defined(__powerc)&&!defined(__MACH__)&&!defined(SYMBIAN) #define POWERPC #define MACOS #define mach_type_known #endif #if defined(OPENBSD)&&defined(__powerpc__) #define POWERPC #define mach_type_known #endif #if defined(DARWIN) #if defined(__ppc__)||defined(__ppc64__) #define POWERPC #define mach_type_known #elif defined(__x86_64__)||defined(__x86_64) #define X86_64 #define mach_type_known #elif defined(__i386__) #define I386 #define mach_type_known #elif defined(__arm__) #define ARM32 #define mach_type_known #elif defined(__aarch64__) #define AARCH64 #define mach_type_known #endif #endif #if defined(__rtems__)&&(defined(i386)||defined(__i386__)) #define I386 #define RTEMS #define mach_type_known #endif #if defined(NeXT)&&defined(mc68000) #define M68K #define NEXT #define mach_type_known #endif #if defined(NeXT)&&(defined(i386)||defined(__i386__)) #define I386 #define NEXT #define mach_type_known #endif #if defined(OPENBSD)&&(defined(i386)||defined(__i386__)) #define I386 #define mach_type_known #endif #if defined(NETBSD)&&(defined(i386)||defined(__i386__)) #define I386 #define mach_type_known #endif #if defined(NETBSD)&&defined(__x86_64__) #define X86_64 #define mach_type_known #endif #if defined(FREEBSD)&&(defined(i386)||defined(__i386__)) #define I386 #define mach_type_known #endif #if (defined(FREEBSD)||defined(SN_TARGET_ORBIS))&&(defined(__amd64__)||defined(__x86_64__)) #define X86_64 #define mach_type_known #endif #if defined(FREEBSD)&&defined(__sparc__) #define SPARC #define mach_type_known #endif #if defined(FREEBSD)&&(defined(powerpc)||defined(__powerpc__)) #define POWERPC #define mach_type_known #endif #if defined(FREEBSD)&&defined(__arm__) #define ARM32 #define mach_type_known #endif #if defined(FREEBSD)&&defined(__aarch64__) #define AARCH64 #define mach_type_known #endif #if defined(FREEBSD)&&(defined(mips)||defined(__mips)||defined(_mips)) #define MIPS #define mach_type_known #endif #if defined(bsdi)&&(defined(i386)||defined(__i386__)) #define I386 #define BSDI #define mach_type_known #endif #if!defined(mach_type_known)&&defined(__386BSD__) #define I386 #define THREE86BSD #define mach_type_known #endif #if defined(_CX_UX)&&defined(_M88K) #define M88K #define CX_UX #define mach_type_known #endif #if defined(DGUX)&&defined(m88k) #define M88K #define mach_type_known #endif #if defined(_WIN32_WCE)||defined(__CEGCC__)||defined(__MINGW32CE__) #if defined(SH3)||defined(SH4) #define SH #endif #if defined(x86)||defined(__i386__) #define I386 #endif #if defined(_M_ARM)||defined(ARM)||defined(_ARM_) #define ARM32 #endif #define MSWINCE #define mach_type_known #else #if ((defined(_MSDOS)||defined(_MSC_VER))&&(_M_IX86>=300))||(defined(_WIN32)&&!defined(__CYGWIN32__)&&!defined(__CYGWIN__)&&!defined(__INTERIX)&&!defined(SYMBIAN)) #if defined(__LP64__)||defined(_M_X64) #define X86_64 #elif defined(_M_ARM) #define ARM32 #elif defined(_M_ARM64) #define AARCH64 #else #define I386 #endif #ifdef _XBOX_ONE #define MSWIN_XBOX1 #else #ifndef MSWIN32 #define MSWIN32 #endif #if defined(WINAPI_FAMILY)&&(WINAPI_FAMILY==WINAPI_FAMILY_APP) #define MSWINRT_FLAVOR #endif #endif #define mach_type_known #endif #if defined(_MSC_VER)&&defined(_M_IA64) #define IA64 #define MSWIN32 #endif #endif #if defined(__DJGPP__) #define I386 #ifndef DJGPP #define DJGPP #endif #define mach_type_known #endif #if defined(__CYGWIN32__)||defined(__CYGWIN__) #if defined(__LP64__) #define X86_64 #else #define I386 #endif #define CYGWIN32 #define mach_type_known #endif #if defined(__INTERIX) #define I386 #define INTERIX #define mach_type_known #endif #if defined(__MINGW32__)&&!defined(mach_type_known) #define I386 #define MSWIN32 #define mach_type_known #endif #if defined(__BORLANDC__) #define I386 #define MSWIN32 #define mach_type_known #endif #if defined(_UTS)&&!defined(mach_type_known) #define S370 #define UTS4 #define mach_type_known #endif #if defined(__pj__)&&!defined(CPPCHECK) #error PicoJava no longer supported #endif #if defined(__embedded__)&&defined(PPC) #define POWERPC #define NOSYS #define mach_type_known #endif #if defined(__WATCOMC__)&&defined(__386__) #define I386 #if!defined(OS2)&&!defined(MSWIN32)&&!defined(DOS4GW) #if defined(__OS2__) #define OS2 #else #if defined(__WINDOWS_386__)||defined(__NT__) #define MSWIN32 #else #define DOS4GW #endif #endif #endif #define mach_type_known #endif #if defined(__s390__)&&defined(LINUX) #define S390 #define mach_type_known #endif #if defined(__GNU__) #if defined(__i386__) #define HURD #define I386 #define mach_type_known #endif #endif #if defined(__TANDEM) #define MIPS #define NONSTOP #define mach_type_known #endif #if defined(__arc__)&&defined(LINUX) #define ARC #define mach_type_known #endif #if defined(__hexagon__)&&defined(LINUX) #define HEXAGON #define mach_type_known #endif #if defined(__tile__)&&defined(LINUX) #ifdef __tilegx__ #define TILEGX #else #define TILEPRO #endif #define mach_type_known #endif #if defined(__riscv)&&(defined(FREEBSD)||defined(LINUX)) #define RISCV #define mach_type_known #endif #if defined(SN_TARGET_PSP2) #define mach_type_known #endif #if defined(NN_PLATFORM_CTR) #define mach_type_known #endif #if defined(NN_BUILD_TARGET_PLATFORM_NX) #define NINTENDO_SWITCH #define mach_type_known #endif #if defined(SYMBIAN) #define mach_type_known #endif #if defined(__EMSCRIPTEN__) #define I386 #define mach_type_known #endif #if!defined(mach_type_known)&&!defined(CPPCHECK) #error The collector has not been ported to this machine/OS combination #endif #if GC_GNUC_PREREQ(2,8)&&!defined(__INTEL_COMPILER)&&!defined(__PATHCC__)&&!defined(__FUJITSU)&&!(defined(POWERPC)&&defined(DARWIN))&&!defined(RTEMS)&&!defined(__ARMCC_VERSION)&&!defined(__clang__) #define HAVE_BUILTIN_UNWIND_INIT #endif #ifdef SYMBIAN #define MACH_TYPE "SYMBIAN" #define OS_TYPE "SYMBIAN" #define CPP_WORDSZ 32 #define ALIGNMENT 4 #define DATASTART (ptr_t)ALIGNMENT #define DATAEND (ptr_t)ALIGNMENT #endif #define STACK_GRAN 0x1000000 #ifdef M68K #define MACH_TYPE "M68K" #define ALIGNMENT 2 #ifdef OPENBSD #define OS_TYPE "OPENBSD" #define HEURISTIC2 #ifdef __ELF__ extern ptr_t GC_data_start; #define DATASTART GC_data_start #define DYNAMIC_LOADING #else extern char etext[]; #define DATASTART ((ptr_t)(etext)) #endif #endif #ifdef NETBSD #define OS_TYPE "NETBSD" #define HEURISTIC2 #ifdef __ELF__ extern ptr_t GC_data_start; #define DATASTART GC_data_start #define DYNAMIC_LOADING #else extern char etext[]; #define DATASTART ((ptr_t)(etext)) #endif #endif #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #if!defined(REDIRECT_MALLOC) #define MPROTECT_VDB #endif #ifdef __ELF__ #define DYNAMIC_LOADING EXTERN_C_END #include EXTERN_C_BEGIN #if defined(__GLIBC__)&&__GLIBC__>=2 #define SEARCH_FOR_DATA_START #else extern char**__environ; #define DATASTART ((ptr_t)(&__environ)) #endif extern int _end[]; #define DATAEND ((ptr_t)(_end)) #else extern int etext[]; #define DATASTART ((ptr_t)((((word)(etext))+0xfff)&~0xfff)) #endif #endif #ifdef AMIGA #define OS_TYPE "AMIGA" #define DATAEND #define GETPAGESIZE()4096 #endif #ifdef MACOS #ifndef __LOWMEM__ EXTERN_C_END #include EXTERN_C_BEGIN #endif #define OS_TYPE "MACOS" #define STACKBOTTOM ((ptr_t)LMGetCurStackBase()) #define DATAEND #define GETPAGESIZE()4096 #endif #ifdef NEXT #define OS_TYPE "NEXT" #define DATASTART ((ptr_t)get_etext()) #define DATASTART_IS_FUNC #define STACKBOTTOM ((ptr_t)0x4000000) #define DATAEND #endif #endif #ifdef POWERPC #define MACH_TYPE "POWERPC" #ifdef MACOS #define ALIGNMENT 2 #ifndef __LOWMEM__ EXTERN_C_END #include EXTERN_C_BEGIN #endif #define OS_TYPE "MACOS" #define STACKBOTTOM ((ptr_t)LMGetCurStackBase()) #define DATAEND #endif #ifdef LINUX #if defined(__powerpc64__) #define ALIGNMENT 8 #define CPP_WORDSZ 64 #ifndef HBLKSIZE #define HBLKSIZE 4096 #endif #else #define ALIGNMENT 4 #endif #define OS_TYPE "LINUX" #if defined(__bg__) #define HEURISTIC2 #define NO_PTHREAD_GETATTR_NP #else #define LINUX_STACKBOTTOM #endif #define COUNT_UNMAPPED_REGIONS #define DYNAMIC_LOADING #define SEARCH_FOR_DATA_START extern int _end[]; #define DATAEND ((ptr_t)(_end)) #endif #ifdef DARWIN #define OS_TYPE "DARWIN" #define DYNAMIC_LOADING #if defined(__ppc64__) #define ALIGNMENT 8 #define CPP_WORDSZ 64 #define STACKBOTTOM ((ptr_t)0x7fff5fc00000) #define CACHE_LINE_SIZE 64 #ifndef HBLKSIZE #define HBLKSIZE 4096 #endif #else #define ALIGNMENT 4 #define STACKBOTTOM ((ptr_t)0xc0000000) #endif #define DATASTART ((ptr_t)get_etext()) #define DATAEND ((ptr_t)get_end()) #define USE_MMAP_ANON #define MPROTECT_VDB EXTERN_C_END #include EXTERN_C_BEGIN #define GETPAGESIZE()(unsigned)getpagesize() #if defined(USE_PPC_PREFETCH)&&defined(__GNUC__) #define PREFETCH(x)__asm__ __volatile__ ("dcbt 0,%0"::"r" ((const void*)(x))) #define GC_PREFETCH_FOR_WRITE(x)__asm__ __volatile__ ("dcbtst 0,%0"::"r" ((const void*)(x))) #endif #define NO_PTHREAD_TRYLOCK #endif #ifdef OPENBSD #define OS_TYPE "OPENBSD" #if defined(__powerpc64__) #define ALIGNMENT 8 #define CPP_WORDSZ 64 #else #define ALIGNMENT 4 #endif #ifndef GC_OPENBSD_THREADS #define HEURISTIC2 #endif extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) extern int _end[]; #define DATAEND ((ptr_t)(&_end)) #define DYNAMIC_LOADING #endif #ifdef FREEBSD #if defined(__powerpc64__) #define ALIGNMENT 8 #define CPP_WORDSZ 64 #ifndef HBLKSIZE #define HBLKSIZE 4096 #endif #else #define ALIGNMENT 4 #endif #define OS_TYPE "FREEBSD" #ifndef GC_FREEBSD_THREADS #define MPROTECT_VDB #endif #define SIG_SUSPEND SIGUSR1 #define SIG_THR_RESTART SIGUSR2 #define FREEBSD_STACKBOTTOM #define DYNAMIC_LOADING extern char etext[]; #define DATASTART GC_FreeBSDGetDataStart(0x1000,(ptr_t)etext) #define DATASTART_USES_BSDGETDATASTART #endif #ifdef NETBSD #define ALIGNMENT 4 #define OS_TYPE "NETBSD" #define HEURISTIC2 extern ptr_t GC_data_start; #define DATASTART GC_data_start #define DYNAMIC_LOADING #endif #ifdef SN_TARGET_PS3 #define OS_TYPE "SN_TARGET_PS3" #define NO_GETENV #define CPP_WORDSZ 32 #define ALIGNMENT 4 extern int _end[]; extern int __bss_start; #define DATASTART ((ptr_t)(__bss_start)) #define DATAEND ((ptr_t)(_end)) #define STACKBOTTOM ((ptr_t)ps3_get_stack_bottom()) #define NO_PTHREAD_TRYLOCK #endif #ifdef AIX #define OS_TYPE "AIX" #undef ALIGNMENT #undef IA64 #ifdef __64BIT__ #define ALIGNMENT 8 #define CPP_WORDSZ 64 #define STACKBOTTOM ((ptr_t)0x1000000000000000) #else #define ALIGNMENT 4 #define CPP_WORDSZ 32 #define STACKBOTTOM ((ptr_t)((ulong)&errno)) #endif #define USE_MMAP_ANON extern int _data[],_end[]; #define DATASTART ((ptr_t)((ulong)_data)) #define DATAEND ((ptr_t)((ulong)_end)) extern int errno; #define DYNAMIC_LOADING #endif #ifdef NOSYS #define ALIGNMENT 4 #define OS_TYPE "NOSYS" extern void __end[],__dso_handle[]; #define DATASTART ((ptr_t)__dso_handle) #define DATAEND ((ptr_t)(__end)) #undef STACK_GRAN #define STACK_GRAN 0x10000000 #define HEURISTIC1 #endif #endif #ifdef NACL #define OS_TYPE "NACL" #if defined(__GLIBC__) #define DYNAMIC_LOADING #endif #define DATASTART ((ptr_t)0x10020000) extern int _end[]; #define DATAEND ((ptr_t)_end) #undef STACK_GRAN #define STACK_GRAN 0x10000 #define HEURISTIC1 #define NO_PTHREAD_GETATTR_NP #define USE_MMAP_ANON #define GETPAGESIZE()65536 #define MAX_NACL_GC_THREADS 1024 #endif #ifdef VAX #define MACH_TYPE "VAX" #define ALIGNMENT 4 extern char etext[]; #define DATASTART ((ptr_t)(etext)) #ifdef BSD #define OS_TYPE "BSD" #define HEURISTIC1 #endif #ifdef ULTRIX #define OS_TYPE "ULTRIX" #define STACKBOTTOM ((ptr_t)0x7fffc800) #endif #endif #ifdef SPARC #define MACH_TYPE "SPARC" #if defined(__arch64__)||defined(__sparcv9) #define ALIGNMENT 8 #define CPP_WORDSZ 64 #define ELF_CLASS ELFCLASS64 #else #define ALIGNMENT 4 #define CPP_WORDSZ 32 #endif #ifdef SOLARIS #define OS_TYPE "SOLARIS" extern int _etext[]; extern int _end[]; ptr_t GC_SysVGetDataStart(size_t,ptr_t); #define DATASTART GC_SysVGetDataStart(0x10000,(ptr_t)_etext) #define DATASTART_IS_FUNC #define DATAEND ((ptr_t)(_end)) #if!defined(USE_MMAP)&&defined(REDIRECT_MALLOC) #define USE_MMAP 1 #endif #ifdef USE_MMAP #define HEAP_START (ptr_t)0x40000000 #else #define HEAP_START DATAEND #endif #define PROC_VDB EXTERN_C_END #include #include EXTERN_C_BEGIN #ifdef USERLIMIT #define STACKBOTTOM ((ptr_t)USRSTACK) #else #define HEURISTIC2 #endif #define GETPAGESIZE()(unsigned)sysconf(_SC_PAGESIZE) #define DYNAMIC_LOADING #endif #ifdef DRSNX #define OS_TYPE "DRSNX" extern int etext[]; ptr_t GC_SysVGetDataStart(size_t,ptr_t); #define DATASTART GC_SysVGetDataStart(0x10000,(ptr_t)etext) #define DATASTART_IS_FUNC #define MPROTECT_VDB #define STACKBOTTOM ((ptr_t)0xdfff0000) #define DYNAMIC_LOADING #endif #ifdef LINUX #define OS_TYPE "LINUX" #ifdef __ELF__ #define DYNAMIC_LOADING #elif!defined(CPPCHECK) #error Linux SPARC a.out not supported #endif #define COUNT_UNMAPPED_REGIONS extern int _end[]; extern int _etext[]; #define DATAEND ((ptr_t)(_end)) #define SVR4 ptr_t GC_SysVGetDataStart(size_t,ptr_t); #ifdef __arch64__ #define DATASTART GC_SysVGetDataStart(0x100000,(ptr_t)_etext) #else #define DATASTART GC_SysVGetDataStart(0x10000,(ptr_t)_etext) #endif #define DATASTART_IS_FUNC #define LINUX_STACKBOTTOM #endif #ifdef OPENBSD #define OS_TYPE "OPENBSD" #ifndef GC_OPENBSD_THREADS #define HEURISTIC2 #endif extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) extern int _end[]; #define DATAEND ((ptr_t)(&_end)) #define DYNAMIC_LOADING #endif #ifdef NETBSD #define OS_TYPE "NETBSD" #define HEURISTIC2 #ifdef __ELF__ extern ptr_t GC_data_start; #define DATASTART GC_data_start #define DYNAMIC_LOADING #else extern char etext[]; #define DATASTART ((ptr_t)(etext)) #endif #endif #ifdef FREEBSD #define OS_TYPE "FREEBSD" #define SIG_SUSPEND SIGUSR1 #define SIG_THR_RESTART SIGUSR2 #define FREEBSD_STACKBOTTOM #define DYNAMIC_LOADING extern char etext[]; extern char edata[]; #if!defined(CPPCHECK) extern char end[]; #endif #define NEED_FIND_LIMIT #define DATASTART ((ptr_t)(&etext)) void*GC_find_limit(void*,int); #define DATAEND (ptr_t)GC_find_limit(DATASTART,TRUE) #define DATAEND_IS_FUNC #define GC_HAVE_DATAREGION2 #define DATASTART2 ((ptr_t)(&edata)) #define DATAEND2 ((ptr_t)(&end)) #endif #endif #ifdef I386 #define MACH_TYPE "I386" #if (defined(__LP64__)||defined(_WIN64))&&!defined(CPPCHECK) #error This should be handled as X86_64 #else #define CPP_WORDSZ 32 #define ALIGNMENT 4 #endif #ifdef SEQUENT #define OS_TYPE "SEQUENT" extern int etext[]; #define DATASTART ((ptr_t)((((word)(etext))+0xfff)&~0xfff)) #define STACKBOTTOM ((ptr_t)0x3ffff000) #endif #if defined(__EMSCRIPTEN__) #define OS_TYPE "EMSCRIPTEN" #define DATASTART (ptr_t)ALIGNMENT #define DATAEND (ptr_t)ALIGNMENT #define USE_MMAP_ANON #define STACK_GROWS_DOWN #endif #if defined(__QNX__) #define OS_TYPE "QNX" #define SA_RESTART 0 #define HEURISTIC1 extern char etext[]; extern int _end[]; #define DATASTART ((ptr_t)etext) #define DATAEND ((ptr_t)_end) #endif #ifdef HAIKU #define OS_TYPE "HAIKU" EXTERN_C_END #include EXTERN_C_BEGIN #define GETPAGESIZE()(unsigned)B_PAGE_SIZE extern int etext[]; #define DATASTART ((ptr_t)((((word)(etext))+0xfff)&~0xfff)) #define DYNAMIC_LOADING #define MPROTECT_VDB #endif #ifdef SOLARIS #define OS_TYPE "SOLARIS" extern int _etext[],_end[]; ptr_t GC_SysVGetDataStart(size_t,ptr_t); #define DATASTART GC_SysVGetDataStart(0x1000,(ptr_t)_etext) #define DATASTART_IS_FUNC #define DATAEND ((ptr_t)(_end)) EXTERN_C_END #include EXTERN_C_BEGIN #ifdef USERLIMIT #define STACKBOTTOM ((ptr_t)USRSTACK) #else #define HEURISTIC2 #endif #ifdef SOLARIS25_PROC_VDB_BUG_FIXED #define PROC_VDB #endif #ifndef GC_THREADS #define MPROTECT_VDB #endif #define DYNAMIC_LOADING #if!defined(USE_MMAP)&&defined(REDIRECT_MALLOC) #define USE_MMAP 1 #endif #ifdef USE_MMAP #define HEAP_START (ptr_t)0x40000000 #else #define HEAP_START DATAEND #endif #endif #ifdef SCO #define OS_TYPE "SCO" extern int etext[]; #define DATASTART ((ptr_t)((((word)(etext))+0x3fffff)&~0x3fffff)+((word)(etext)&0xfff)) #define STACKBOTTOM ((ptr_t)0x7ffffffc) #endif #ifdef SCO_ELF #define OS_TYPE "SCO_ELF" extern int etext[]; #define DATASTART ((ptr_t)(etext)) #define STACKBOTTOM ((ptr_t)0x08048000) #define DYNAMIC_LOADING #define ELF_CLASS ELFCLASS32 #endif #ifdef DGUX #define OS_TYPE "DGUX" extern int _etext,_end; ptr_t GC_SysVGetDataStart(size_t,ptr_t); #define DATASTART GC_SysVGetDataStart(0x1000,(ptr_t)(&_etext)) #define DATASTART_IS_FUNC #define DATAEND ((ptr_t)(&_end)) #define STACK_GROWS_DOWN #define HEURISTIC2 EXTERN_C_END #include EXTERN_C_BEGIN #define GETPAGESIZE()(unsigned)sysconf(_SC_PAGESIZE) #define DYNAMIC_LOADING #ifndef USE_MMAP #define USE_MMAP 1 #endif #define MAP_FAILED (void*)((word)-1) #define HEAP_START (ptr_t)0x40000000 #endif #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #if!defined(REDIRECT_MALLOC) #define MPROTECT_VDB #else #endif #define HEAP_START (ptr_t)0x1000 #ifdef __ELF__ #define DYNAMIC_LOADING EXTERN_C_END #include EXTERN_C_BEGIN #if defined(__GLIBC__)&&__GLIBC__>=2||defined(HOST_ANDROID)||defined(HOST_TIZEN) #define SEARCH_FOR_DATA_START #else extern char**__environ; #define DATASTART ((ptr_t)(&__environ)) #endif extern int _end[]; #define DATAEND ((ptr_t)(_end)) #if!defined(GC_NO_SIGSETJMP)&&(defined(HOST_TIZEN)||(defined(HOST_ANDROID)&&!(GC_GNUC_PREREQ(4,8)||GC_CLANG_PREREQ(3,2)||__ANDROID_API__>=18))) #define GC_NO_SIGSETJMP 1 #endif #else extern int etext[]; #define DATASTART ((ptr_t)((((word)(etext))+0xfff)&~0xfff)) #endif #ifdef USE_I686_PREFETCH #define PREFETCH(x)__asm__ __volatile__ ("prefetchnta %0"::"m"(*(char*)(x))) #ifdef FORCE_WRITE_PREFETCH #define GC_PREFETCH_FOR_WRITE(x)__asm__ __volatile__ ("prefetcht0 %0"::"m"(*(char*)(x))) #else #define GC_NO_PREFETCH_FOR_WRITE #endif #elif defined(USE_3DNOW_PREFETCH) #define PREFETCH(x)__asm__ __volatile__ ("prefetch %0"::"m"(*(char*)(x))) #define GC_PREFETCH_FOR_WRITE(x)__asm__ __volatile__ ("prefetchw %0"::"m"(*(char*)(x))) #endif #if defined(__GLIBC__)&&!defined(__UCLIBC__) #define GLIBC_2_19_TSX_BUG EXTERN_C_END #include EXTERN_C_BEGIN #endif #endif #ifdef CYGWIN32 #define OS_TYPE "CYGWIN32" #define WOW64_THREAD_CONTEXT_WORKAROUND #define RETRY_GET_THREAD_CONTEXT #define DATASTART ((ptr_t)GC_DATASTART) #define DATAEND ((ptr_t)GC_DATAEND) #ifdef USE_WINALLOC #define GWW_VDB #else # #ifdef USE_MMAP #define NEED_FIND_LIMIT #define USE_MMAP_ANON #endif #endif #endif #ifdef INTERIX #define OS_TYPE "INTERIX" extern int _data_start__[]; extern int _bss_end__[]; #define DATASTART ((ptr_t)_data_start__) #define DATAEND ((ptr_t)_bss_end__) #define STACKBOTTOM ({ ptr_t rv;__asm__ __volatile__ ("movl %%fs:4,%%eax":"=a" (rv));rv;}) #define USE_MMAP_ANON #endif #ifdef OS2 #define OS_TYPE "OS2" #define DATAEND #endif #ifdef MSWIN32 #define OS_TYPE "MSWIN32" #define WOW64_THREAD_CONTEXT_WORKAROUND #define RETRY_GET_THREAD_CONTEXT #define MPROTECT_VDB #define GWW_VDB #define DATAEND #endif #ifdef MSWINCE #define OS_TYPE "MSWINCE" #define DATAEND #endif #ifdef DJGPP #define OS_TYPE "DJGPP" EXTERN_C_END #include "stubinfo.h" EXTERN_C_BEGIN extern int etext[]; extern int _stklen; extern int __djgpp_stack_limit; #define DATASTART ((ptr_t)((((word)(etext))+0x1ff)&~0x1ff)) #define STACKBOTTOM ((ptr_t)((word)__djgpp_stack_limit+_stklen)) #endif #ifdef OPENBSD #define OS_TYPE "OPENBSD" #ifndef GC_OPENBSD_THREADS #define HEURISTIC2 #endif extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) extern int _end[]; #define DATAEND ((ptr_t)(&_end)) #define DYNAMIC_LOADING #endif #ifdef FREEBSD #define OS_TYPE "FREEBSD" #ifndef GC_FREEBSD_THREADS #define MPROTECT_VDB #endif #ifdef __GLIBC__ #define SIG_SUSPEND (32+6) #define SIG_THR_RESTART (32+5) extern int _end[]; #define DATAEND ((ptr_t)(_end)) #else #define SIG_SUSPEND SIGUSR1 #define SIG_THR_RESTART SIGUSR2 #endif #define FREEBSD_STACKBOTTOM #ifdef __ELF__ #define DYNAMIC_LOADING #endif extern char etext[]; #define DATASTART GC_FreeBSDGetDataStart(0x1000,(ptr_t)etext) #define DATASTART_USES_BSDGETDATASTART #endif #ifdef NETBSD #define OS_TYPE "NETBSD" #ifdef __ELF__ #define DYNAMIC_LOADING #endif #endif #ifdef THREE86BSD #define OS_TYPE "THREE86BSD" #endif #ifdef BSDI #define OS_TYPE "BSDI" #endif #if defined(NETBSD)||defined(THREE86BSD)||defined(BSDI) #define HEURISTIC2 extern char etext[]; #define DATASTART ((ptr_t)(etext)) #endif #ifdef NEXT #define OS_TYPE "NEXT" #define DATASTART ((ptr_t)get_etext()) #define DATASTART_IS_FUNC #define STACKBOTTOM ((ptr_t)0xc0000000) #define DATAEND #endif #ifdef RTEMS #define OS_TYPE "RTEMS" EXTERN_C_END #include EXTERN_C_BEGIN extern int etext[]; void*rtems_get_stack_bottom(void); #define InitStackBottom rtems_get_stack_bottom() #define DATASTART ((ptr_t)etext) #define STACKBOTTOM ((ptr_t)InitStackBottom) #define SIG_SUSPEND SIGUSR1 #define SIG_THR_RESTART SIGUSR2 #endif #ifdef DOS4GW #define OS_TYPE "DOS4GW" extern long __nullarea; extern char _end; extern char*_STACKTOP; #pragma aux __nullarea "*"; #pragma aux _end "*"; #define STACKBOTTOM ((ptr_t)_STACKTOP) #define DATASTART ((ptr_t)(&__nullarea)) #define DATAEND ((ptr_t)(&_end)) #endif #ifdef HURD #define OS_TYPE "HURD" #define STACK_GROWS_DOWN #define HEURISTIC2 #define SIG_SUSPEND SIGUSR1 #define SIG_THR_RESTART SIGUSR2 #define SEARCH_FOR_DATA_START extern int _end[]; #define DATAEND ((ptr_t)(_end)) #define DYNAMIC_LOADING #define USE_MMAP_ANON #endif #ifdef DARWIN #define OS_TYPE "DARWIN" #define DARWIN_DONT_PARSE_STACK 1 #define DYNAMIC_LOADING #define DATASTART ((ptr_t)get_etext()) #define DATAEND ((ptr_t)get_end()) #define STACKBOTTOM ((ptr_t)0xc0000000) #define USE_MMAP_ANON #define MPROTECT_VDB EXTERN_C_END #include EXTERN_C_BEGIN #define GETPAGESIZE()(unsigned)getpagesize() #define NO_PTHREAD_TRYLOCK #if TARGET_OS_IPHONE&&!defined(NO_DYLD_BIND_FULLY_IMAGE) #define NO_DYLD_BIND_FULLY_IMAGE #endif #endif #endif #ifdef NS32K #define MACH_TYPE "NS32K" #define ALIGNMENT 4 extern char**environ; #define DATASTART ((ptr_t)(&environ)) #define STACKBOTTOM ((ptr_t)0xfffff000) #endif #ifdef MIPS #define MACH_TYPE "MIPS" #ifdef LINUX #define OS_TYPE "LINUX" #define DYNAMIC_LOADING #define COUNT_UNMAPPED_REGIONS extern int _end[]; #pragma weak __data_start extern int __data_start[]; #define DATASTART ((ptr_t)(__data_start)) #define DATAEND ((ptr_t)(_end)) #ifdef _MIPS_SZPTR #define CPP_WORDSZ _MIPS_SZPTR #define ALIGNMENT (_MIPS_SZPTR/8) #else #define ALIGNMENT 4 #endif #ifndef HBLKSIZE #define HBLKSIZE 4096 #endif #if __GLIBC__==2&&__GLIBC_MINOR__>=2||__GLIBC__ > 2 #define LINUX_STACKBOTTOM #else #define STACKBOTTOM ((ptr_t)0x7fff8000) #endif #endif #ifdef EWS4800 #define HEURISTIC2 #if defined(_MIPS_SZPTR)&&(_MIPS_SZPTR==64) extern int _fdata[],_end[]; #define DATASTART ((ptr_t)_fdata) #define DATAEND ((ptr_t)_end) #define CPP_WORDSZ _MIPS_SZPTR #define ALIGNMENT (_MIPS_SZPTR/8) #else extern int etext[],edata[]; #if!defined(CPPCHECK) extern int end[]; #endif extern int _DYNAMIC_LINKING[],_gp[]; #define DATASTART ((ptr_t)((((word)(etext)+0x3ffff)&~0x3ffff)+((word)(etext)&0xffff))) #define DATAEND ((ptr_t)(edata)) #define GC_HAVE_DATAREGION2 #define DATASTART2 (_DYNAMIC_LINKING?(ptr_t)(((word)_gp+0x8000+0x3ffff)&~0x3ffff):(ptr_t)edata) #define DATAEND2 ((ptr_t)(end)) #define ALIGNMENT 4 #endif #define OS_TYPE "EWS4800" #endif #ifdef ULTRIX #define HEURISTIC2 #define DATASTART ((ptr_t)0x10000000) #define OS_TYPE "ULTRIX" #define ALIGNMENT 4 #endif #ifdef IRIX5 #define HEURISTIC2 extern int _fdata[]; #define DATASTART ((ptr_t)(_fdata)) #ifdef USE_MMAP #define HEAP_START (ptr_t)0x30000000 #else #define HEAP_START DATASTART #endif #define OS_TYPE "IRIX5" #ifdef _MIPS_SZPTR #define CPP_WORDSZ _MIPS_SZPTR #define ALIGNMENT (_MIPS_SZPTR/8) #else #define ALIGNMENT 4 #endif #define DYNAMIC_LOADING #endif #ifdef MSWINCE #define OS_TYPE "MSWINCE" #define ALIGNMENT 4 #define DATAEND #endif #ifdef NETBSD #define OS_TYPE "NETBSD" #define ALIGNMENT 4 #define HEURISTIC2 #ifdef __ELF__ extern ptr_t GC_data_start; #define DATASTART GC_data_start #define NEED_FIND_LIMIT #define DYNAMIC_LOADING #else #define DATASTART ((ptr_t)0x10000000) #define STACKBOTTOM ((ptr_t)0x7ffff000) #endif #endif #ifdef OPENBSD #define OS_TYPE "OPENBSD" #define CPP_WORDSZ 64 #define ALIGNMENT 8 #ifndef GC_OPENBSD_THREADS #define HEURISTIC2 #endif extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) extern int _end[]; #define DATAEND ((ptr_t)(&_end)) #define DYNAMIC_LOADING #endif #ifdef FREEBSD #define OS_TYPE "FREEBSD" #define ALIGNMENT 4 #ifndef GC_FREEBSD_THREADS #define MPROTECT_VDB #endif #define SIG_SUSPEND SIGUSR1 #define SIG_THR_RESTART SIGUSR2 #define FREEBSD_STACKBOTTOM #define DYNAMIC_LOADING extern char etext[]; #define DATASTART GC_FreeBSDGetDataStart(0x1000,(ptr_t)etext) #define DATASTART_USES_BSDGETDATASTART #endif #if defined(NONSTOP) #define CPP_WORDSZ 32 #define OS_TYPE "NONSTOP" #define ALIGNMENT 4 #define DATASTART ((ptr_t)0x08000000) extern char**environ; #define DATAEND ((ptr_t)(environ - 0x10)) #define STACKBOTTOM ((ptr_t)0x4fffffff) #endif #endif #ifdef NIOS2 #define CPP_WORDSZ 32 #define MACH_TYPE "NIOS2" #ifdef LINUX #define OS_TYPE "LINUX" #define DYNAMIC_LOADING #define COUNT_UNMAPPED_REGIONS extern int _end[]; extern int __data_start[]; #define DATASTART ((ptr_t)(__data_start)) #define DATAEND ((ptr_t)(_end)) #define ALIGNMENT 4 #ifndef HBLKSIZE #define HBLKSIZE 4096 #endif #define LINUX_STACKBOTTOM #endif #endif #ifdef OR1K #define CPP_WORDSZ 32 #define MACH_TYPE "OR1K" #ifdef LINUX #define OS_TYPE "LINUX" #define DYNAMIC_LOADING #define COUNT_UNMAPPED_REGIONS extern int _end[]; extern int __data_start[]; #define DATASTART ((ptr_t)(__data_start)) #define DATAEND ((ptr_t)(_end)) #define ALIGNMENT 4 #ifndef HBLKSIZE #define HBLKSIZE 4096 #endif #define LINUX_STACKBOTTOM #endif #endif #ifdef HP_PA #define MACH_TYPE "HP_PA" #ifdef __LP64__ #define CPP_WORDSZ 64 #define ALIGNMENT 8 #else #define CPP_WORDSZ 32 #define ALIGNMENT 4 #endif #if!defined(GC_HPUX_THREADS)&&!defined(GC_LINUX_THREADS)&&!defined(OPENBSD)&&!defined(LINUX) #define MPROTECT_VDB #endif #define STACK_GROWS_UP #ifdef HPUX #define OS_TYPE "HPUX" extern int __data_start[]; #define DATASTART ((ptr_t)(__data_start)) #ifdef USE_MMAP #define USE_MMAP_ANON #endif #ifdef USE_HPUX_FIXED_STACKBOTTOM #define STACKBOTTOM ((ptr_t)0x7b033000) #elif defined(USE_ENVIRON_POINTER) extern char**environ; #define STACKBOTTOM ((ptr_t)environ) #elif!defined(HEURISTIC2) #define HPUX_MAIN_STACKBOTTOM #define NEED_FIND_LIMIT #endif #define DYNAMIC_LOADING EXTERN_C_END #include EXTERN_C_BEGIN #define GETPAGESIZE()(unsigned)sysconf(_SC_PAGE_SIZE) #ifndef __GNUC__ #define PREFETCH(x)do { register long addr=(long)(x);(void)_asm ("LDW",0,0,addr,0);} while (0) #endif #endif #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #define DYNAMIC_LOADING #define SEARCH_FOR_DATA_START extern int _end[]; #define DATAEND ((ptr_t)(&_end)) #endif #ifdef OPENBSD #define OS_TYPE "OPENBSD" #ifndef GC_OPENBSD_THREADS #define HEURISTIC2 #endif extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) extern int _end[]; #define DATAEND ((ptr_t)(&_end)) #define DYNAMIC_LOADING #endif #endif #ifdef ALPHA #define MACH_TYPE "ALPHA" #define ALIGNMENT 8 #define CPP_WORDSZ 64 #ifdef NETBSD #define OS_TYPE "NETBSD" #define HEURISTIC2 extern ptr_t GC_data_start; #define DATASTART GC_data_start #define ELFCLASS32 32 #define ELFCLASS64 64 #define ELF_CLASS ELFCLASS64 #define DYNAMIC_LOADING #endif #ifdef OPENBSD #define OS_TYPE "OPENBSD" #ifndef GC_OPENBSD_THREADS #define HEURISTIC2 #endif extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) extern int _end[]; #define DATAEND ((ptr_t)(&_end)) #define DYNAMIC_LOADING #endif #ifdef FREEBSD #define OS_TYPE "FREEBSD" #define SIG_SUSPEND SIGUSR1 #define SIG_THR_RESTART SIGUSR2 #define FREEBSD_STACKBOTTOM #define DYNAMIC_LOADING extern char etext[]; extern char edata[]; #if!defined(CPPCHECK) extern char end[]; #endif #define NEED_FIND_LIMIT #define DATASTART ((ptr_t)(&etext)) void*GC_find_limit(void*,int); #define DATAEND (ptr_t)GC_find_limit(DATASTART,TRUE) #define DATAEND_IS_FUNC #define GC_HAVE_DATAREGION2 #define DATASTART2 ((ptr_t)(&edata)) #define DATAEND2 ((ptr_t)(&end)) #endif #ifdef OSF1 #define OS_TYPE "OSF1" #define DATASTART ((ptr_t)0x140000000) extern int _end[]; #define DATAEND ((ptr_t)(&_end)) extern char**environ; #define STACKBOTTOM ((ptr_t)(((word)(environ)|(getpagesize()-1))+1)) extern int __start[]; #define HEURISTIC2_LIMIT ((ptr_t)((word)(__start)&~(getpagesize()-1))) #ifndef GC_OSF1_THREADS #define MPROTECT_VDB #endif #define DYNAMIC_LOADING #endif #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #ifdef __ELF__ #define SEARCH_FOR_DATA_START #define DYNAMIC_LOADING #else #define DATASTART ((ptr_t)0x140000000) #endif extern int _end[]; #define DATAEND ((ptr_t)(_end)) #if!defined(REDIRECT_MALLOC) #define MPROTECT_VDB #endif #endif #endif #ifdef IA64 #define MACH_TYPE "IA64" #ifdef HPUX #ifdef _ILP32 #define CPP_WORDSZ 32 #define ALIGNMENT 4 #else #if!defined(_LP64)&&!defined(CPPCHECK) #error Unknown ABI #endif #define CPP_WORDSZ 64 #define ALIGNMENT 8 #endif #define OS_TYPE "HPUX" extern int __data_start[]; #define DATASTART ((ptr_t)(__data_start)) #ifdef USE_MMAP #define USE_MMAP_ANON #endif extern char**environ; #define STACKBOTTOM ((ptr_t)environ) #define HPUX_STACKBOTTOM #define DYNAMIC_LOADING EXTERN_C_END #include EXTERN_C_BEGIN #define GETPAGESIZE()(unsigned)sysconf(_SC_PAGE_SIZE) #define BACKING_STORE_DISPLACEMENT 0x1000000 #define BACKING_STORE_ALIGNMENT 0x1000 extern ptr_t GC_register_stackbottom; #define BACKING_STORE_BASE GC_register_stackbottom #endif #ifdef LINUX #define CPP_WORDSZ 64 #define ALIGNMENT 8 #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM extern ptr_t GC_register_stackbottom; #define BACKING_STORE_BASE GC_register_stackbottom #define COUNT_UNMAPPED_REGIONS #define SEARCH_FOR_DATA_START #ifdef __GNUC__ #define DYNAMIC_LOADING #else #endif #if!defined(REDIRECT_MALLOC) #define MPROTECT_VDB #endif extern int _end[]; #define DATAEND ((ptr_t)(_end)) #ifdef __GNUC__ #ifndef __INTEL_COMPILER #define PREFETCH(x)__asm__ (" lfetch [%0]"::"r"(x)) #define GC_PREFETCH_FOR_WRITE(x)__asm__ (" lfetch.excl [%0]"::"r"(x)) #define CLEAR_DOUBLE(x)__asm__ (" stf.spill [%0]=f0"::"r"((void*)(x))) #else EXTERN_C_END #include EXTERN_C_BEGIN #define PREFETCH(x)__lfetch(__lfhint_none,(x)) #define GC_PREFETCH_FOR_WRITE(x)__lfetch(__lfhint_nta,(x)) #define CLEAR_DOUBLE(x)__stf_spill((void*)(x),0) #endif #endif #endif #ifdef MSWIN32 #define OS_TYPE "MSWIN32" #define DATAEND #if defined(_WIN64) #define CPP_WORDSZ 64 #else #define CPP_WORDSZ 32 #endif #define ALIGNMENT 8 #endif #endif #ifdef M88K #define MACH_TYPE "M88K" #define ALIGNMENT 4 extern int etext[]; #ifdef CX_UX #define OS_TYPE "CX_UX" #define DATASTART ((ptr_t)((((word)(etext)+0x3fffff)&~0x3fffff)+0x10000)) #endif #ifdef DGUX #define OS_TYPE "DGUX" ptr_t GC_SysVGetDataStart(size_t,ptr_t); #define DATASTART GC_SysVGetDataStart(0x10000,(ptr_t)etext) #define DATASTART_IS_FUNC #endif #define STACKBOTTOM ((char*)0xf0000000) #endif #ifdef S370 #define MACH_TYPE "S370" #define ALIGNMENT 4 #ifdef UTS4 #define OS_TYPE "UTS4" extern int etext[]; extern int _etext[]; extern int _end[]; ptr_t GC_SysVGetDataStart(size_t,ptr_t); #define DATASTART GC_SysVGetDataStart(0x10000,(ptr_t)_etext) #define DATASTART_IS_FUNC #define DATAEND ((ptr_t)(_end)) #define HEURISTIC2 #endif #endif #ifdef S390 #define MACH_TYPE "S390" #ifndef __s390x__ #define ALIGNMENT 4 #define CPP_WORDSZ 32 #else #define ALIGNMENT 8 #define CPP_WORDSZ 64 #ifndef HBLKSIZE #define HBLKSIZE 4096 #endif #endif #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #define DYNAMIC_LOADING #define COUNT_UNMAPPED_REGIONS extern int __data_start[] __attribute__((__weak__)); #define DATASTART ((ptr_t)(__data_start)) extern int _end[] __attribute__((__weak__)); #define DATAEND ((ptr_t)(_end)) #define CACHE_LINE_SIZE 256 #define GETPAGESIZE()4096 #endif #endif #ifdef AARCH64 #define MACH_TYPE "AARCH64" #ifdef __ILP32__ #define CPP_WORDSZ 32 #define ALIGNMENT 4 #else #define CPP_WORDSZ 64 #define ALIGNMENT 8 #endif #ifndef HBLKSIZE #define HBLKSIZE 4096 #endif #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #if!defined(REDIRECT_MALLOC) #define MPROTECT_VDB #endif #define DYNAMIC_LOADING #if defined(HOST_ANDROID) #define SEARCH_FOR_DATA_START #else extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) #endif extern int _end[]; #define DATAEND ((ptr_t)(&_end)) #endif #ifdef DARWIN #define OS_TYPE "DARWIN" #define DARWIN_DONT_PARSE_STACK 1 #define DYNAMIC_LOADING #define DATASTART ((ptr_t)get_etext()) #define DATAEND ((ptr_t)get_end()) #define STACKBOTTOM ((ptr_t)0x16fdfffff) #define USE_MMAP_ANON EXTERN_C_END #include EXTERN_C_BEGIN #define GETPAGESIZE()(unsigned)getpagesize() #define NO_PTHREAD_TRYLOCK #if TARGET_OS_IPHONE&&!defined(NO_DYLD_BIND_FULLY_IMAGE) #define NO_DYLD_BIND_FULLY_IMAGE #endif #endif #ifdef FREEBSD #define OS_TYPE "FREEBSD" #ifndef GC_FREEBSD_THREADS #define MPROTECT_VDB #endif #define FREEBSD_STACKBOTTOM #define DYNAMIC_LOADING extern char etext[]; #define DATASTART GC_FreeBSDGetDataStart(0x1000,(ptr_t)etext) #define DATASTART_USES_BSDGETDATASTART #endif #ifdef NETBSD #define OS_TYPE "NETBSD" #define HEURISTIC2 extern ptr_t GC_data_start; #define DATASTART GC_data_start #define ELF_CLASS ELFCLASS64 #define DYNAMIC_LOADING #endif #ifdef OPENBSD #define OS_TYPE "OPENBSD" #ifndef GC_OPENBSD_THREADS #define HEURISTIC2 #endif extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) extern int _end[]; #define DATAEND ((ptr_t)(&_end)) #define DYNAMIC_LOADING #endif #ifdef NINTENDO_SWITCH #define OS_TYPE "NINTENDO_SWITCH" extern int __bss_end[]; #define NO_HANDLE_FORK 1 #define DATASTART (ptr_t)ALIGNMENT #define DATAEND (ptr_t)(&__bss_end) void*switch_get_stack_bottom(void); #define STACKBOTTOM ((ptr_t)switch_get_stack_bottom()) #endif #ifdef MSWIN32 #define OS_TYPE "MSWIN32" #ifndef DATAEND #define DATAEND #endif #endif #ifdef NOSYS extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) extern void*__stack_base__; #define STACKBOTTOM ((ptr_t)__stack_base__) #endif #endif #ifdef ARM32 #if defined(NACL) #define MACH_TYPE "NACL" #else #define MACH_TYPE "ARM32" #endif #define CPP_WORDSZ 32 #define ALIGNMENT 4 #ifdef NETBSD #define OS_TYPE "NETBSD" #define HEURISTIC2 #ifdef __ELF__ extern ptr_t GC_data_start; #define DATASTART GC_data_start #define DYNAMIC_LOADING #else extern char etext[]; #define DATASTART ((ptr_t)(etext)) #endif #endif #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #if!defined(REDIRECT_MALLOC) #define MPROTECT_VDB #endif #define DYNAMIC_LOADING EXTERN_C_END #include EXTERN_C_BEGIN #if defined(__GLIBC__)&&__GLIBC__>=2||defined(HOST_ANDROID)||defined(HOST_TIZEN) #define SEARCH_FOR_DATA_START #else extern char**__environ; #define DATASTART ((ptr_t)(&__environ)) #endif extern int _end[]; #define DATAEND ((ptr_t)(_end)) #endif #ifdef MSWINCE #define OS_TYPE "MSWINCE" #define DATAEND #endif #ifdef FREEBSD #define OS_TYPE "FREEBSD" #ifndef GC_FREEBSD_THREADS #define MPROTECT_VDB #endif #define SIG_SUSPEND SIGUSR1 #define SIG_THR_RESTART SIGUSR2 #define FREEBSD_STACKBOTTOM #define DYNAMIC_LOADING extern char etext[]; #define DATASTART GC_FreeBSDGetDataStart(0x1000,(ptr_t)etext) #define DATASTART_USES_BSDGETDATASTART #endif #ifdef DARWIN #define OS_TYPE "DARWIN" #define DARWIN_DONT_PARSE_STACK 1 #define DYNAMIC_LOADING #define DATASTART ((ptr_t)get_etext()) #define DATAEND ((ptr_t)get_end()) #define STACKBOTTOM ((ptr_t)0x30000000) #define USE_MMAP_ANON EXTERN_C_END #include EXTERN_C_BEGIN #define GETPAGESIZE()(unsigned)getpagesize() #define NO_PTHREAD_TRYLOCK #if TARGET_OS_IPHONE&&!defined(NO_DYLD_BIND_FULLY_IMAGE) #define NO_DYLD_BIND_FULLY_IMAGE #endif #endif #ifdef OPENBSD #define OS_TYPE "OPENBSD" #ifndef GC_OPENBSD_THREADS #define HEURISTIC2 #endif extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) extern int _end[]; #define DATAEND ((ptr_t)(&_end)) #define DYNAMIC_LOADING #endif #ifdef SN_TARGET_PSP2 #define OS_TYPE "SN_TARGET_PSP2" #define NO_HANDLE_FORK 1 #define DATASTART (ptr_t)ALIGNMENT #define DATAEND (ptr_t)ALIGNMENT void*psp2_get_stack_bottom(void); #define STACKBOTTOM ((ptr_t)psp2_get_stack_bottom()) #endif #ifdef NN_PLATFORM_CTR #define OS_TYPE "NN_PLATFORM_CTR" extern unsigned char Image$$ZI$$ZI$$Base[]; #define DATASTART (ptr_t)(Image$$ZI$$ZI$$Base) extern unsigned char Image$$ZI$$ZI$$Limit[]; #define DATAEND (ptr_t)(Image$$ZI$$ZI$$Limit) void*n3ds_get_stack_bottom(void); #define STACKBOTTOM ((ptr_t)n3ds_get_stack_bottom()) #endif #ifdef MSWIN32 #define OS_TYPE "MSWIN32" #ifndef DATAEND #define DATAEND #endif #endif #ifdef NOSYS extern int __data_start[]; #define DATASTART ((ptr_t)(__data_start)) extern void*__stack_base__; #define STACKBOTTOM ((ptr_t)(__stack_base__)) #endif #endif #ifdef CRIS #define MACH_TYPE "CRIS" #define CPP_WORDSZ 32 #define ALIGNMENT 1 #ifdef LINUX #define OS_TYPE "LINUX" #define DYNAMIC_LOADING #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #define SEARCH_FOR_DATA_START extern int _end[]; #define DATAEND ((ptr_t)(_end)) #endif #endif #if defined(SH)&&!defined(SH4) #define MACH_TYPE "SH" #define ALIGNMENT 4 #ifdef MSWINCE #define OS_TYPE "MSWINCE" #define DATAEND #endif #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #define DYNAMIC_LOADING #define SEARCH_FOR_DATA_START extern int _end[]; #define DATAEND ((ptr_t)(_end)) #endif #ifdef NETBSD #define OS_TYPE "NETBSD" #define HEURISTIC2 extern ptr_t GC_data_start; #define DATASTART GC_data_start #define DYNAMIC_LOADING #endif #ifdef OPENBSD #define OS_TYPE "OPENBSD" #ifndef GC_OPENBSD_THREADS #define HEURISTIC2 #endif extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) extern int _end[]; #define DATAEND ((ptr_t)(&_end)) #define DYNAMIC_LOADING #endif #endif #ifdef SH4 #define MACH_TYPE "SH4" #define ALIGNMENT 4 #ifdef MSWINCE #define OS_TYPE "MSWINCE" #define DATAEND #endif #endif #ifdef AVR32 #define MACH_TYPE "AVR32" #define CPP_WORDSZ 32 #define ALIGNMENT 4 #ifdef LINUX #define OS_TYPE "LINUX" #define DYNAMIC_LOADING #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #define SEARCH_FOR_DATA_START extern int _end[]; #define DATAEND ((ptr_t)(_end)) #endif #endif #ifdef M32R #define CPP_WORDSZ 32 #define MACH_TYPE "M32R" #define ALIGNMENT 4 #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #define DYNAMIC_LOADING #define SEARCH_FOR_DATA_START extern int _end[]; #define DATAEND ((ptr_t)(_end)) #endif #endif #ifdef X86_64 #define MACH_TYPE "X86_64" #ifdef __ILP32__ #define ALIGNMENT 4 #define CPP_WORDSZ 32 #else #define ALIGNMENT 8 #define CPP_WORDSZ 64 #endif #ifndef HBLKSIZE #define HBLKSIZE 4096 #endif #ifndef CACHE_LINE_SIZE #define CACHE_LINE_SIZE 64 #endif #ifdef SN_TARGET_ORBIS #define OS_TYPE "SN_TARGET_ORBIS" #define DATASTART (ptr_t)ALIGNMENT #define DATAEND (ptr_t)ALIGNMENT void*ps4_get_stack_bottom(void); #define STACKBOTTOM ((ptr_t)ps4_get_stack_bottom()) #endif #ifdef OPENBSD #define OS_TYPE "OPENBSD" #ifndef GC_OPENBSD_THREADS #define HEURISTIC2 #endif extern int __data_start[]; extern int _end[]; #define DATASTART ((ptr_t)__data_start) #define DATAEND ((ptr_t)(&_end)) #define DYNAMIC_LOADING #endif #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #if!defined(REDIRECT_MALLOC) #define MPROTECT_VDB #else #endif #define COUNT_UNMAPPED_REGIONS #define DYNAMIC_LOADING EXTERN_C_END #include EXTERN_C_BEGIN #define SEARCH_FOR_DATA_START extern int _end[]; #define DATAEND ((ptr_t)(_end)) #if defined(__GLIBC__)&&!defined(__UCLIBC__) #define USE_MMAP_ANON #define GETCONTEXT_FPU_EXCMASK_BUG #define GLIBC_2_19_TSX_BUG EXTERN_C_END #include EXTERN_C_BEGIN #endif #endif #ifdef DARWIN #define OS_TYPE "DARWIN" #define DARWIN_DONT_PARSE_STACK 1 #define DYNAMIC_LOADING #define DATASTART ((ptr_t)get_etext()) #define DATAEND ((ptr_t)get_end()) #define STACKBOTTOM ((ptr_t)0x7fff5fc00000) #define USE_MMAP_ANON #define MPROTECT_VDB EXTERN_C_END #include EXTERN_C_BEGIN #define GETPAGESIZE()(unsigned)getpagesize() #define NO_PTHREAD_TRYLOCK #if TARGET_OS_IPHONE&&!defined(NO_DYLD_BIND_FULLY_IMAGE) #define NO_DYLD_BIND_FULLY_IMAGE #endif #endif #ifdef FREEBSD #define OS_TYPE "FREEBSD" #ifndef GC_FREEBSD_THREADS #define MPROTECT_VDB #endif #ifdef __GLIBC__ #define SIG_SUSPEND (32+6) #define SIG_THR_RESTART (32+5) extern int _end[]; #define DATAEND ((ptr_t)(_end)) #else #define SIG_SUSPEND SIGUSR1 #define SIG_THR_RESTART SIGUSR2 #endif #define FREEBSD_STACKBOTTOM #if defined(__DragonFly__) #define COUNT_UNMAPPED_REGIONS #endif #define DYNAMIC_LOADING extern char etext[]; #define DATASTART GC_FreeBSDGetDataStart(0x1000,(ptr_t)etext) #define DATASTART_USES_BSDGETDATASTART #endif #ifdef NETBSD #define OS_TYPE "NETBSD" #define HEURISTIC2 extern ptr_t GC_data_start; #define DATASTART GC_data_start #define DYNAMIC_LOADING #endif #ifdef HAIKU #define OS_TYPE "HAIKU" EXTERN_C_END #include EXTERN_C_BEGIN #define GETPAGESIZE()(unsigned)B_PAGE_SIZE #define HEURISTIC2 #define SEARCH_FOR_DATA_START #define DYNAMIC_LOADING #define MPROTECT_VDB #endif #ifdef SOLARIS #define OS_TYPE "SOLARIS" #define ELF_CLASS ELFCLASS64 extern int _etext[],_end[]; ptr_t GC_SysVGetDataStart(size_t,ptr_t); #define DATASTART GC_SysVGetDataStart(0x1000,(ptr_t)_etext) #define DATASTART_IS_FUNC #define DATAEND ((ptr_t)(_end)) EXTERN_C_END #include EXTERN_C_BEGIN #ifdef USERLIMIT #define STACKBOTTOM ((ptr_t)USRSTACK) #else #define HEURISTIC2 #endif #ifdef SOLARIS25_PROC_VDB_BUG_FIXED #define PROC_VDB #endif #ifndef GC_THREADS #define MPROTECT_VDB #endif #define DYNAMIC_LOADING #if!defined(USE_MMAP)&&defined(REDIRECT_MALLOC) #define USE_MMAP 1 #endif #ifdef USE_MMAP #define HEAP_START (ptr_t)0x40000000 #else #define HEAP_START DATAEND #endif #endif #ifdef CYGWIN32 #define OS_TYPE "CYGWIN32" #define RETRY_GET_THREAD_CONTEXT #ifdef USE_WINALLOC #define GWW_VDB #else #if defined(THREAD_LOCAL_ALLOC) #else #define MPROTECT_VDB #endif #ifdef USE_MMAP #define USE_MMAP_ANON #endif #endif #endif #ifdef MSWIN_XBOX1 #define OS_TYPE "MSWIN_XBOX1" #define NO_GETENV #define DATASTART (ptr_t)ALIGNMENT #define DATAEND (ptr_t)ALIGNMENT LONG64 durango_get_stack_bottom(void); #define STACKBOTTOM ((ptr_t)durango_get_stack_bottom()) #define GETPAGESIZE()4096 #ifndef USE_MMAP #define USE_MMAP 1 #endif #define PROT_NONE 0 #define PROT_READ 1 #define PROT_WRITE 2 #define PROT_EXEC 4 #define MAP_PRIVATE 2 #define MAP_FIXED 0x10 #define MAP_FAILED ((void*)-1) #endif #ifdef MSWIN32 #define OS_TYPE "MSWIN32" #define RETRY_GET_THREAD_CONTEXT #if!defined(__GNUC__)||defined(__INTEL_COMPILER)||GC_GNUC_PREREQ(4,7) #define MPROTECT_VDB #endif #define GWW_VDB #ifndef DATAEND #define DATAEND #endif #endif #endif #ifdef ARC #define CPP_WORDSZ 32 #define MACH_TYPE "ARC" #define ALIGNMENT 4 #define CACHE_LINE_SIZE 64 #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #define DYNAMIC_LOADING extern int __data_start[] __attribute__((__weak__)); #define DATASTART ((ptr_t)__data_start) #endif #endif #ifdef HEXAGON #define CPP_WORDSZ 32 #define MACH_TYPE "HEXAGON" #define ALIGNMENT 4 #ifdef LINUX #define OS_TYPE "LINUX" #define LINUX_STACKBOTTOM #if!defined(REDIRECT_MALLOC) #define MPROTECT_VDB #endif #define COUNT_UNMAPPED_REGIONS #define DYNAMIC_LOADING EXTERN_C_END #include EXTERN_C_BEGIN #if defined(__GLIBC__) #define SEARCH_FOR_DATA_START #elif!defined(CPPCHECK) #error Unknown Hexagon libc configuration #endif extern int _end[]; #define DATAEND ((ptr_t)(_end)) #endif #endif #ifdef TILEPRO #define CPP_WORDSZ 32 #define MACH_TYPE "TILEPro" #define ALIGNMENT 4 #define PREFETCH(x)__insn_prefetch(x) #define CACHE_LINE_SIZE 64 #ifdef LINUX #define OS_TYPE "LINUX" extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #define DYNAMIC_LOADING #endif #endif #ifdef TILEGX #define CPP_WORDSZ (__SIZEOF_POINTER__*8) #define MACH_TYPE "TILE-Gx" #define ALIGNMENT __SIZEOF_POINTER__ #if CPP_WORDSZ < 64 #define CLEAR_DOUBLE(x)(*(long long*)(x)=0) #endif #define PREFETCH(x)__insn_prefetch_l1(x) #define CACHE_LINE_SIZE 64 #ifdef LINUX #define OS_TYPE "LINUX" extern int __data_start[]; #define DATASTART ((ptr_t)__data_start) #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #define DYNAMIC_LOADING #endif #endif #ifdef RISCV #define MACH_TYPE "RISC-V" #define CPP_WORDSZ __riscv_xlen #define ALIGNMENT (CPP_WORDSZ/8) #ifdef FREEBSD #define OS_TYPE "FREEBSD" #ifndef GC_FREEBSD_THREADS #define MPROTECT_VDB #endif #define SIG_SUSPEND SIGUSR1 #define SIG_THR_RESTART SIGUSR2 #define FREEBSD_STACKBOTTOM #define DYNAMIC_LOADING extern char etext[]; #define DATASTART GC_FreeBSDGetDataStart(0x1000,(ptr_t)etext) #define DATASTART_USES_BSDGETDATASTART #endif #ifdef LINUX #define OS_TYPE "LINUX" extern int __data_start[] __attribute__((__weak__)); #define DATASTART ((ptr_t)__data_start) #define LINUX_STACKBOTTOM #define COUNT_UNMAPPED_REGIONS #define DYNAMIC_LOADING #endif #endif #if defined(__GLIBC__)&&!defined(DONT_USE_LIBC_PRIVATES) #define USE_LIBC_PRIVATES #endif #ifdef NO_RETRY_GET_THREAD_CONTEXT #undef RETRY_GET_THREAD_CONTEXT #endif #if defined(LINUX_STACKBOTTOM)&&defined(NO_PROC_STAT)&&!defined(USE_LIBC_PRIVATES) #undef LINUX_STACKBOTTOM #define HEURISTIC2 #endif #if defined(USE_MMAP_ANON)&&!defined(USE_MMAP) #define USE_MMAP 1 #elif (defined(LINUX)||defined(OPENBSD))&&defined(USE_MMAP) #define USE_MMAP_ANON #endif #if defined(GC_LINUX_THREADS)&&defined(REDIRECT_MALLOC)&&!defined(USE_PROC_FOR_LIBRARIES) #define USE_PROC_FOR_LIBRARIES #endif #ifndef STACK_GROWS_UP #define STACK_GROWS_DOWN #endif #ifndef CPP_WORDSZ #define CPP_WORDSZ 32 #endif #ifndef OS_TYPE #define OS_TYPE "" #endif #ifndef DATAEND #if!defined(CPPCHECK) extern int end[]; #endif #define DATAEND ((ptr_t)(end)) #endif #if defined(HOST_ANDROID)&&defined(__clang__)&&!defined(BROKEN_UUENDUU_SYM) #undef DATAEND #pragma weak __end__ extern int __end__[]; #define DATAEND (__end__!=0?(ptr_t)__end__:(ptr_t)_end) #endif #if (defined(SVR4)||defined(HOST_ANDROID)||defined(HOST_TIZEN))&&!defined(GETPAGESIZE) EXTERN_C_END #include EXTERN_C_BEGIN #define GETPAGESIZE()(unsigned)sysconf(_SC_PAGESIZE) #endif #ifndef GETPAGESIZE #if defined(SOLARIS)||defined(IRIX5)||defined(LINUX)||defined(NETBSD)||defined(FREEBSD)||defined(HPUX) EXTERN_C_END #include EXTERN_C_BEGIN #endif #define GETPAGESIZE()(unsigned)getpagesize() #endif #if defined(HOST_ANDROID)&&!(__ANDROID_API__>=23)&&((defined(MIPS)&&(CPP_WORDSZ==32))||defined(ARM32)||defined(I386)) #define USE_TKILL_ON_ANDROID #endif #if defined(SOLARIS)||defined(DRSNX)||defined(UTS4) #define SVR4 #endif #if defined(SOLARIS)||defined(DRSNX) #define SOLARISDL #define SUNOS5SIGS #endif #if defined(HPUX) #define SUNOS5SIGS #endif #if defined(FREEBSD)&&(defined(__DragonFly__)||__FreeBSD__>=4||(__FreeBSD_kernel__>=4)) #define SUNOS5SIGS #endif #if!defined(GC_EXPLICIT_SIGNALS_UNBLOCK)&&defined(SUNOS5SIGS)&&!defined(GC_NO_PTHREAD_SIGMASK) #define GC_EXPLICIT_SIGNALS_UNBLOCK #endif #if!defined(NO_SIGNALS_UNBLOCK_IN_MAIN)&&defined(GC_NO_PTHREAD_SIGMASK) #define NO_SIGNALS_UNBLOCK_IN_MAIN #endif #if!defined(NO_MARKER_SPECIAL_SIGMASK)&&(defined(NACL)||defined(GC_WIN32_PTHREADS)) #define NO_MARKER_SPECIAL_SIGMASK #endif #ifdef GC_NETBSD_THREADS #define SIGRTMIN 33 #define SIGRTMAX 63 #define GC_NETBSD_THREADS_WORKAROUND #endif #ifdef GC_OPENBSD_THREADS EXTERN_C_END #include EXTERN_C_BEGIN #if OpenBSD < 201211 #define GC_OPENBSD_UTHREADS 1 #endif #endif #if defined(SVR4)||defined(LINUX)||defined(IRIX5)||defined(HPUX)||defined(OPENBSD)||defined(NETBSD)||defined(FREEBSD)||defined(DGUX)||defined(BSD)||defined(HAIKU)||defined(HURD)||defined(AIX)||defined(DARWIN)||defined(OSF1) #define UNIX_LIKE #endif #if defined(CPPCHECK) #undef CPP_WORDSZ #define CPP_WORDSZ (__SIZEOF_POINTER__*8) #elif CPP_WORDSZ!=32&&CPP_WORDSZ!=64 #error Bad word size #endif #if!defined(ALIGNMENT)&&!defined(CPPCHECK) #error Undefined ALIGNMENT #endif #ifdef PCR #undef DYNAMIC_LOADING #undef STACKBOTTOM #undef HEURISTIC1 #undef HEURISTIC2 #undef PROC_VDB #undef MPROTECT_VDB #define PCR_VDB #endif #if!defined(STACKBOTTOM)&&(defined(ECOS)||defined(NOSYS))&&!defined(CPPCHECK) #error Undefined STACKBOTTOM #endif #ifdef IGNORE_DYNAMIC_LOADING #undef DYNAMIC_LOADING #endif #if defined(SMALL_CONFIG)&&!defined(GC_DISABLE_INCREMENTAL) #define GC_DISABLE_INCREMENTAL #endif #if (defined(MSWIN32)||defined(MSWINCE))&&!defined(USE_WINALLOC) #define USE_WINALLOC 1 #endif #ifdef USE_WINALLOC #undef USE_MMAP #endif #if defined(DARWIN)||defined(FREEBSD)||defined(HAIKU)||defined(IRIX5)||defined(LINUX)||defined(NETBSD)||defined(OPENBSD)||defined(SOLARIS)||((defined(CYGWIN32)||defined(USE_MMAP)||defined(USE_MUNMAP))&&!defined(USE_WINALLOC)) #define MMAP_SUPPORTED #endif #if defined(USE_MUNMAP)&&!defined(MUNMAP_THRESHOLD)&&(defined(SN_TARGET_ORBIS)||defined(SN_TARGET_PS3)||defined(SN_TARGET_PSP2)||defined(MSWIN_XBOX1)) #define MUNMAP_THRESHOLD 2 #endif #if defined(USE_MUNMAP)&&defined(COUNT_UNMAPPED_REGIONS)&&!defined(GC_UNMAPPED_REGIONS_SOFT_LIMIT) #if defined(__DragonFly__) #define GC_UNMAPPED_REGIONS_SOFT_LIMIT (1000000/4) #else #define GC_UNMAPPED_REGIONS_SOFT_LIMIT 16384 #endif #endif #if defined(GC_DISABLE_INCREMENTAL)||defined(DEFAULT_VDB) #undef GWW_VDB #undef MPROTECT_VDB #undef PCR_VDB #undef PROC_VDB #endif #ifdef GC_DISABLE_INCREMENTAL #undef CHECKSUMS #endif #ifdef USE_GLOBAL_ALLOC #undef GWW_VDB #endif #if defined(BASE_ATOMIC_OPS_EMULATED) #undef MPROTECT_VDB #endif #if defined(USE_PROC_FOR_LIBRARIES)&&defined(GC_LINUX_THREADS) #undef MPROTECT_VDB #endif #if defined(MPROTECT_VDB)&&defined(GC_PREFER_MPROTECT_VDB) #undef PCR_VDB #undef PROC_VDB #endif #ifdef PROC_VDB #undef MPROTECT_VDB #endif #if defined(MPROTECT_VDB)&&!defined(MSWIN32)&&!defined(MSWINCE) #include #endif #if defined(SIGBUS)&&!defined(HAVE_SIGBUS)&&!defined(CPPCHECK) #define HAVE_SIGBUS #endif #ifndef SA_SIGINFO #define NO_SA_SIGACTION #endif #if (defined(NO_SA_SIGACTION)||defined(GC_NO_SIGSETJMP))&&defined(MPROTECT_VDB)&&!defined(DARWIN)&&!defined(MSWIN32)&&!defined(MSWINCE) #undef MPROTECT_VDB #endif #if!defined(PCR_VDB)&&!defined(PROC_VDB)&&!defined(MPROTECT_VDB)&&!defined(GWW_VDB)&&!defined(DEFAULT_VDB)&&!defined(GC_DISABLE_INCREMENTAL) #define DEFAULT_VDB #endif #if ((defined(UNIX_LIKE)&&(defined(DARWIN)||defined(HAIKU)||defined(HURD)||defined(OPENBSD)||defined(ARM32)||defined(AVR32)||defined(MIPS)||defined(NIOS2)||defined(OR1K)))||(defined(LINUX)&&!defined(__gnu_linux__))||(defined(RTEMS)&&defined(I386))||defined(HOST_ANDROID))&&!defined(NO_GETCONTEXT) #define NO_GETCONTEXT 1 #endif #ifndef PREFETCH #if GC_GNUC_PREREQ(3,0)&&!defined(NO_PREFETCH) #define PREFETCH(x)__builtin_prefetch((x),0,0) #else #define PREFETCH(x)(void)0 #endif #endif #ifndef GC_PREFETCH_FOR_WRITE #if GC_GNUC_PREREQ(3,0)&&!defined(GC_NO_PREFETCH_FOR_WRITE) #define GC_PREFETCH_FOR_WRITE(x)__builtin_prefetch((x),1) #else #define GC_PREFETCH_FOR_WRITE(x)(void)0 #endif #endif #ifndef CACHE_LINE_SIZE #define CACHE_LINE_SIZE 32 #endif #ifndef STATIC #ifdef GC_ASSERTIONS #define STATIC #else #define STATIC static #endif #endif #if defined(LINUX)&&(defined(USE_PROC_FOR_LIBRARIES)||defined(IA64)||!defined(SMALL_CONFIG)) #define NEED_PROC_MAPS #endif #if defined(LINUX)||defined(HURD)||defined(__GLIBC__) #define REGISTER_LIBRARIES_EARLY #endif #if defined(SEARCH_FOR_DATA_START) extern ptr_t GC_data_start; #define DATASTART GC_data_start #endif #ifndef HEAP_START #define HEAP_START ((ptr_t)0) #endif #ifndef CLEAR_DOUBLE #define CLEAR_DOUBLE(x)(((word*)(x))[0]=0,((word*)(x))[1]=0) #endif #if defined(GC_LINUX_THREADS)&&defined(REDIRECT_MALLOC)&&!defined(INCLUDE_LINUX_THREAD_DESCR) #define INCLUDE_LINUX_THREAD_DESCR #endif #if!defined(CPPCHECK) #if defined(GC_IRIX_THREADS)&&!defined(IRIX5) #error Inconsistent configuration #endif #if defined(GC_LINUX_THREADS)&&!defined(LINUX)&&!defined(NACL) #error Inconsistent configuration #endif #if defined(GC_NETBSD_THREADS)&&!defined(NETBSD) #error Inconsistent configuration #endif #if defined(GC_FREEBSD_THREADS)&&!defined(FREEBSD) #error Inconsistent configuration #endif #if defined(GC_SOLARIS_THREADS)&&!defined(SOLARIS) #error Inconsistent configuration #endif #if defined(GC_HPUX_THREADS)&&!defined(HPUX) #error Inconsistent configuration #endif #if defined(GC_AIX_THREADS)&&!defined(_AIX) #error Inconsistent configuration #endif #if defined(GC_WIN32_THREADS)&&!defined(CYGWIN32)&&!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(MSWIN_XBOX1) #error Inconsistent configuration #endif #if defined(GC_WIN32_PTHREADS)&&defined(CYGWIN32) #error Inconsistent configuration #endif #endif #if defined(PCR)||defined(GC_WIN32_THREADS)||defined(GC_PTHREADS)||((defined(NN_PLATFORM_CTR)||defined(NINTENDO_SWITCH)||defined(SN_TARGET_ORBIS)||defined(SN_TARGET_PS3)||defined(SN_TARGET_PSP2))&&defined(GC_THREADS)) #define THREADS #endif #if defined(PARALLEL_MARK)&&!defined(THREADS)&&!defined(CPPCHECK) #error Invalid config:PARALLEL_MARK requires GC_THREADS #endif #if defined(GWW_VDB)&&!defined(USE_WINALLOC)&&!defined(CPPCHECK) #error Invalid config:GWW_VDB requires USE_WINALLOC #endif #if (((defined(MSWIN32)||defined(MSWINCE))&&!defined(__GNUC__))||(defined(MSWIN32)&&defined(I386))||(defined(USE_PROC_FOR_LIBRARIES)&&defined(THREADS)))&&!defined(NO_CRT)&&!defined(NO_WRAP_MARK_SOME) #define WRAP_MARK_SOME #endif #if defined(PARALLEL_MARK)&&!defined(DEFAULT_STACK_MAYBE_SMALL)&&(defined(HPUX)||defined(GC_DGUX386_THREADS)||defined(NO_GETCONTEXT)) #define DEFAULT_STACK_MAYBE_SMALL #endif #ifdef PARALLEL_MARK #define MIN_STACK_SIZE (8*HBLKSIZE*sizeof(word)) #endif #if defined(HOST_ANDROID)&&!defined(THREADS)&&!defined(USE_GET_STACKBASE_FOR_MAIN) #define USE_GET_STACKBASE_FOR_MAIN #endif #if ((defined(FREEBSD)&&defined(__GLIBC__))||defined(LINUX)||defined(NETBSD)||defined(HOST_ANDROID))&&!defined(NO_PTHREAD_GETATTR_NP) #define HAVE_PTHREAD_GETATTR_NP 1 #elif defined(FREEBSD)&&!defined(__GLIBC__)&&!defined(NO_PTHREAD_ATTR_GET_NP) #define HAVE_PTHREAD_NP_H 1 #define HAVE_PTHREAD_ATTR_GET_NP 1 #endif #if defined(UNIX_LIKE)&&defined(THREADS)&&!defined(NO_CANCEL_SAFE)&&!defined(HOST_ANDROID) #define CANCEL_SAFE #endif #ifdef CANCEL_SAFE #define IF_CANCEL(x)x #else #define IF_CANCEL(x) #endif #if!defined(CAN_HANDLE_FORK)&&!defined(NO_HANDLE_FORK)&&!defined(HAVE_NO_FORK)&&((defined(GC_PTHREADS)&&!defined(NACL)&&!defined(GC_WIN32_PTHREADS)&&!defined(USE_WINALLOC))||(defined(DARWIN)&&defined(MPROTECT_VDB))||defined(HANDLE_FORK)) #define CAN_HANDLE_FORK #endif #if defined(CAN_HANDLE_FORK)&&!defined(CAN_CALL_ATFORK)&&!defined(HURD)&&!defined(SN_TARGET_ORBIS)&&!defined(HOST_TIZEN)&&(!defined(HOST_ANDROID)||__ANDROID_API__>=21) #define CAN_CALL_ATFORK #endif #if!defined(CAN_HANDLE_FORK)&&!defined(HAVE_NO_FORK)&&(defined(MSWIN32)||defined(MSWINCE)||defined(DOS4GW)||defined(OS2)||defined(SYMBIAN)) #define HAVE_NO_FORK #endif #if!defined(USE_MARK_BITS)&&!defined(USE_MARK_BYTES)&&defined(PARALLEL_MARK) #define USE_MARK_BYTES #endif #if (defined(MSWINCE)&&!defined(__CEGCC__)||defined(MSWINRT_FLAVOR))&&!defined(NO_GETENV) #define NO_GETENV #endif #if (defined(NO_GETENV)||defined(MSWINCE))&&!defined(NO_GETENV_WIN32) #define NO_GETENV_WIN32 #endif #if!defined(MSGBOX_ON_ERROR)&&!defined(NO_MSGBOX_ON_ERROR)&&!defined(SMALL_CONFIG)&&defined(MSWIN32)&&!defined(MSWINRT_FLAVOR)&&!defined(MSWIN_XBOX1) #define MSGBOX_ON_ERROR #endif #ifndef STRTOULL #if defined(_WIN64)&&!defined(__GNUC__) #define STRTOULL _strtoui64 #elif defined(_LLP64)||defined(__LLP64__)||defined(_WIN64) #define STRTOULL strtoull #else #define STRTOULL strtoul #endif #endif #ifndef GC_WORD_C #if defined(_WIN64)&&!defined(__GNUC__) #define GC_WORD_C(val)val##ui64 #elif defined(_LLP64)||defined(__LLP64__)||defined(_WIN64) #define GC_WORD_C(val)val##ULL #else #define GC_WORD_C(val)((word)val##UL) #endif #endif #if defined(__has_feature) #if __has_feature(address_sanitizer)&&!defined(ADDRESS_SANITIZER) #define ADDRESS_SANITIZER #endif #if __has_feature(memory_sanitizer)&&!defined(MEMORY_SANITIZER) #define MEMORY_SANITIZER #endif #if __has_feature(thread_sanitizer)&&!defined(THREAD_SANITIZER) #define THREAD_SANITIZER #endif #else #ifdef __SANITIZE_ADDRESS__ #define ADDRESS_SANITIZER #endif #endif #if defined(SPARC) #define ASM_CLEAR_CODE #endif #if defined(SPARC) #define CAN_SAVE_CALL_ARGS #endif #if (defined(I386)||defined(X86_64))&&(defined(LINUX)||defined(__GLIBC__)) #define CAN_SAVE_CALL_ARGS #endif #if defined(SAVE_CALL_COUNT)&&!defined(GC_ADD_CALLER)&&defined(GC_CAN_SAVE_CALL_STACKS) #define SAVE_CALL_CHAIN #endif #ifdef SAVE_CALL_CHAIN #if defined(SAVE_CALL_NARGS)&&defined(CAN_SAVE_CALL_ARGS) #define NARGS SAVE_CALL_NARGS #else #define NARGS 0 #endif #endif #ifdef SAVE_CALL_CHAIN #if!defined(SAVE_CALL_COUNT)||defined(CPPCHECK) #define NFRAMES 6 #else #define NFRAMES ((SAVE_CALL_COUNT+1)&~1) #endif #define NEED_CALLINFO #endif #ifdef GC_ADD_CALLER #define NFRAMES 1 #define NARGS 0 #define NEED_CALLINFO #endif #if (defined(FREEBSD)||(defined(DARWIN)&&!defined(_POSIX_C_SOURCE))||(defined(SOLARIS)&&(!defined(_XOPEN_SOURCE)||defined(__EXTENSIONS__)))||defined(LINUX))&&!defined(HAVE_DLADDR) #define HAVE_DLADDR 1 #endif #if defined(MAKE_BACK_GRAPH)&&!defined(DBG_HDRS_ALL) #define DBG_HDRS_ALL 1 #endif #if defined(POINTER_MASK)&&!defined(POINTER_SHIFT) #define POINTER_SHIFT 0 #endif #if defined(POINTER_SHIFT)&&!defined(POINTER_MASK) #define POINTER_MASK ((word)(-1)) #endif #if!defined(FIXUP_POINTER)&&defined(POINTER_MASK) #define FIXUP_POINTER(p)(p=((p)&POINTER_MASK)< 32 #define HASH_TL #endif #if defined(LARGE_CONFIG)||!defined(SMALL_CONFIG) #define LOG_BOTTOM_SZ 10 #else #define LOG_BOTTOM_SZ 11 #endif #define BOTTOM_SZ (1<>LOG_HBLKSIZE)&(HDR_CACHE_SIZE-1))) #define HCE_VALID_FOR(hce,h)((hce)->block_addr==((word)(h)>>LOG_HBLKSIZE)) #define HCE_HDR(h)((hce)->hce_hdr) #ifdef PRINT_BLACK_LIST GC_INNER hdr*GC_header_cache_miss(ptr_t p,hdr_cache_entry*hce, ptr_t source); #define HEADER_CACHE_MISS(p,hce,source)GC_header_cache_miss(p,hce,source) #else GC_INNER hdr*GC_header_cache_miss(ptr_t p,hdr_cache_entry*hce); #define HEADER_CACHE_MISS(p,hce,source)GC_header_cache_miss(p,hce) #endif #define HC_GET_HDR(p,hhdr,source){ hdr_cache_entry*hce=HCE(p);if (EXPECT(HCE_VALID_FOR(hce,p),TRUE)){ HC_HIT();hhdr=hce->hce_hdr;} else { hhdr=HEADER_CACHE_MISS(p,hce,source);if (NULL==hhdr)break;} } typedef struct bi { hdr*index[BOTTOM_SZ]; struct bi*asc_link; struct bi*desc_link; word key; #ifdef HASH_TL struct bi*hash_link; #endif } bottom_index; #define MAX_JUMP (HBLKSIZE - 1) #define HDR_FROM_BI(bi,p)((bi)->index[((word)(p)>>LOG_HBLKSIZE)&(BOTTOM_SZ - 1)]) #ifndef HASH_TL #define BI(p)(GC_top_index [(word)(p)>>(LOG_BOTTOM_SZ+LOG_HBLKSIZE)]) #define HDR_INNER(p)HDR_FROM_BI(BI(p),p) #ifdef SMALL_CONFIG #define HDR(p)GC_find_header((ptr_t)(p)) #else #define HDR(p)HDR_INNER(p) #endif #define GET_BI(p,bottom_indx)(void)((bottom_indx)=BI(p)) #define GET_HDR(p,hhdr)(void)((hhdr)=HDR(p)) #define SET_HDR(p,hhdr)(void)(HDR_INNER(p)=(hhdr)) #define GET_HDR_ADDR(p,ha)(void)((ha)=&HDR_INNER(p)) #else #define TL_HASH(hi)((hi)&(TOP_SZ - 1)) #define GET_BI(p,bottom_indx)do { REGISTER word hi=(word)(p)>>(LOG_BOTTOM_SZ+LOG_HBLKSIZE);REGISTER bottom_index*_bi=GC_top_index[TL_HASH(hi)];while (_bi->key!=hi&&_bi!=GC_all_nils)_bi=_bi->hash_link;(bottom_indx)=_bi;} while (0) #define GET_HDR_ADDR(p,ha)do { REGISTER bottom_index*bi;GET_BI(p,bi);(ha)=&HDR_FROM_BI(bi,p);} while (0) #define GET_HDR(p,hhdr)do { REGISTER hdr**_ha;GET_HDR_ADDR(p,_ha);(hhdr)=*_ha;} while (0) #define SET_HDR(p,hhdr)do { REGISTER hdr**_ha;GET_HDR_ADDR(p,_ha);*_ha=(hhdr);} while (0) #define HDR(p)GC_find_header((ptr_t)(p)) #endif #define IS_FORWARDING_ADDR_OR_NIL(hhdr)((size_t)(hhdr)<=MAX_JUMP) #define FORWARDED_ADDR(h,hhdr)((struct hblk*)(h)- (size_t)(hhdr)) EXTERN_C_END #endif #endif #ifndef GC_ATTR_NO_SANITIZE_ADDR #ifndef ADDRESS_SANITIZER #define GC_ATTR_NO_SANITIZE_ADDR #elif GC_CLANG_PREREQ(3,8) #define GC_ATTR_NO_SANITIZE_ADDR __attribute__((no_sanitize("address"))) #else #define GC_ATTR_NO_SANITIZE_ADDR __attribute__((no_sanitize_address)) #endif #endif #ifndef GC_ATTR_NO_SANITIZE_MEMORY #ifndef MEMORY_SANITIZER #define GC_ATTR_NO_SANITIZE_MEMORY #elif GC_CLANG_PREREQ(3,8) #define GC_ATTR_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory"))) #else #define GC_ATTR_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) #endif #endif #ifndef GC_ATTR_NO_SANITIZE_THREAD #ifndef THREAD_SANITIZER #define GC_ATTR_NO_SANITIZE_THREAD #elif GC_CLANG_PREREQ(3,8) #define GC_ATTR_NO_SANITIZE_THREAD __attribute__((no_sanitize("thread"))) #else #define GC_ATTR_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread)) #endif #endif #ifndef GC_ATTR_UNUSED #if GC_GNUC_PREREQ(3,4) #define GC_ATTR_UNUSED __attribute__((__unused__)) #else #define GC_ATTR_UNUSED #endif #endif #ifdef HAVE_CONFIG_H #define GC_INLINE static inline #elif defined(_MSC_VER)||defined(__INTEL_COMPILER)||defined(__DMC__)||(GC_GNUC_PREREQ(3,0)&&defined(__STRICT_ANSI__))||defined(__WATCOMC__) #define GC_INLINE static __inline #elif GC_GNUC_PREREQ(3,0)||defined(__sun) #define GC_INLINE static inline #else #define GC_INLINE static #endif #ifndef GC_ATTR_NOINLINE #if GC_GNUC_PREREQ(4,0) #define GC_ATTR_NOINLINE __attribute__((__noinline__)) #elif _MSC_VER>=1400 #define GC_ATTR_NOINLINE __declspec(noinline) #else #define GC_ATTR_NOINLINE #endif #endif #ifndef GC_API_OSCALL #if defined(__GNUC__) #if GC_GNUC_PREREQ(4,0)&&!defined(GC_NO_VISIBILITY) #define GC_API_OSCALL extern __attribute__((__visibility__("default"))) #else #define GC_API_OSCALL extern #endif #else #define GC_API_OSCALL GC_API #endif #endif #ifndef GC_API_PRIV #define GC_API_PRIV GC_API #endif #if defined(THREADS)&&!defined(NN_PLATFORM_CTR) #ifndef GC_ATOMIC_OPS_H #define GC_ATOMIC_OPS_H #ifdef GC_BUILTIN_ATOMIC #ifdef __cplusplus extern "C" { #endif typedef GC_word AO_t; #ifdef GC_PRIVATE_H #define AO_INLINE GC_INLINE #else #define AO_INLINE static __inline #endif typedef unsigned char AO_TS_t; #define AO_TS_CLEAR 0 #define AO_TS_INITIALIZER (AO_TS_t)AO_TS_CLEAR #if defined(__GCC_ATOMIC_TEST_AND_SET_TRUEVAL)&&!defined(CPPCHECK) #define AO_TS_SET __GCC_ATOMIC_TEST_AND_SET_TRUEVAL #else #define AO_TS_SET (AO_TS_t)1 #endif #define AO_CLEAR(p)__atomic_clear(p,__ATOMIC_RELEASE) #define AO_test_and_set_acquire(p)__atomic_test_and_set(p,__ATOMIC_ACQUIRE) #define AO_HAVE_test_and_set_acquire #define AO_compiler_barrier()__atomic_signal_fence(__ATOMIC_SEQ_CST) #define AO_nop_full()__atomic_thread_fence(__ATOMIC_SEQ_CST) #define AO_HAVE_nop_full #define AO_fetch_and_add(p,v)__atomic_fetch_add(p,v,__ATOMIC_RELAXED) #define AO_HAVE_fetch_and_add #define AO_fetch_and_add1(p)AO_fetch_and_add(p,1) #define AO_HAVE_fetch_and_add1 #define AO_or(p,v)(void)__atomic_or_fetch(p,v,__ATOMIC_RELAXED) #define AO_HAVE_or #define AO_load(p)__atomic_load_n(p,__ATOMIC_RELAXED) #define AO_HAVE_load #define AO_load_acquire(p)__atomic_load_n(p,__ATOMIC_ACQUIRE) #define AO_HAVE_load_acquire #define AO_load_acquire_read(p)AO_load_acquire(p) #define AO_HAVE_load_acquire_read #define AO_store(p,v)__atomic_store_n(p,v,__ATOMIC_RELAXED) #define AO_HAVE_store #define AO_store_release(p,v)__atomic_store_n(p,v,__ATOMIC_RELEASE) #define AO_HAVE_store_release #define AO_store_release_write(p,v)AO_store_release(p,v) #define AO_HAVE_store_release_write #define AO_char_load(p)__atomic_load_n(p,__ATOMIC_RELAXED) #define AO_HAVE_char_load #define AO_char_store(p,v)__atomic_store_n(p,v,__ATOMIC_RELAXED) #define AO_HAVE_char_store #ifdef AO_REQUIRE_CAS AO_INLINE int AO_compare_and_swap(volatile AO_t*p,AO_t ov,AO_t nv) { return (int)__atomic_compare_exchange_n(p,&ov,nv,0, __ATOMIC_RELAXED,__ATOMIC_RELAXED); } AO_INLINE int AO_compare_and_swap_release(volatile AO_t*p,AO_t ov,AO_t nv) { return (int)__atomic_compare_exchange_n(p,&ov,nv,0, __ATOMIC_RELEASE,__ATOMIC_RELAXED); } #define AO_HAVE_compare_and_swap_release #endif #ifdef __cplusplus } #endif #ifndef NO_LOCKFREE_AO_OR #define HAVE_LOCKFREE_AO_OR 1 #endif #else #include "atomic_ops.h" #if (!defined(AO_HAVE_load)||!defined(AO_HAVE_store))&&!defined(CPPCHECK) #error AO_load or AO_store is missing;probably old version of atomic_ops #endif #endif #endif #ifndef AO_HAVE_compiler_barrier #define AO_HAVE_compiler_barrier 1 #endif #endif #if defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32) #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN 1 #endif #define NOSERVICE #include #include #endif #ifndef GC_LOCKS_H #define GC_LOCKS_H #ifdef THREADS #ifdef PCR #include #include #endif EXTERN_C_BEGIN #ifdef PCR GC_EXTERN PCR_Th_ML GC_allocate_ml; #if defined(CPPCHECK) #define DCL_LOCK_STATE #else #define DCL_LOCK_STATE PCR_ERes GC_fastLockRes;PCR_sigset_t GC_old_sig_mask #endif #define UNCOND_LOCK()PCR_Th_ML_Acquire(&GC_allocate_ml) #define UNCOND_UNLOCK()PCR_Th_ML_Release(&GC_allocate_ml) #elif defined(NN_PLATFORM_CTR)||defined(NINTENDO_SWITCH) extern void GC_lock(void); extern void GC_unlock(void); #define UNCOND_LOCK()GC_lock() #define UNCOND_UNLOCK()GC_unlock() #endif #if (!defined(AO_HAVE_test_and_set_acquire)||defined(GC_RTEMS_PTHREADS)||defined(SN_TARGET_ORBIS)||defined(SN_TARGET_PS3)||defined(GC_WIN32_THREADS)||defined(BASE_ATOMIC_OPS_EMULATED)||defined(LINT2))&&defined(GC_PTHREADS) #define USE_PTHREAD_LOCKS #undef USE_SPIN_LOCK #endif #if defined(GC_WIN32_THREADS)&&!defined(USE_PTHREAD_LOCKS) #define NO_THREAD (DWORD)(-1) GC_EXTERN CRITICAL_SECTION GC_allocate_ml; #ifdef GC_ASSERTIONS GC_EXTERN DWORD GC_lock_holder; #define SET_LOCK_HOLDER()GC_lock_holder=GetCurrentThreadId() #define UNSET_LOCK_HOLDER()GC_lock_holder=NO_THREAD #define I_HOLD_LOCK()(!GC_need_to_lock||GC_lock_holder==GetCurrentThreadId()) #ifdef THREAD_SANITIZER #define I_DONT_HOLD_LOCK()TRUE #else #define I_DONT_HOLD_LOCK()(!GC_need_to_lock||GC_lock_holder!=GetCurrentThreadId()) #endif #define UNCOND_LOCK(){ GC_ASSERT(I_DONT_HOLD_LOCK());EnterCriticalSection(&GC_allocate_ml);SET_LOCK_HOLDER();} #define UNCOND_UNLOCK(){ GC_ASSERT(I_HOLD_LOCK());UNSET_LOCK_HOLDER();LeaveCriticalSection(&GC_allocate_ml);} #else #define UNCOND_LOCK()EnterCriticalSection(&GC_allocate_ml) #define UNCOND_UNLOCK()LeaveCriticalSection(&GC_allocate_ml) #endif #elif defined(GC_PTHREADS) EXTERN_C_END #include EXTERN_C_BEGIN #if!defined(GC_WIN32_PTHREADS) #define NUMERIC_THREAD_ID(id)((unsigned long)(id)) #define THREAD_EQUAL(id1,id2)((id1)==(id2)) #define NUMERIC_THREAD_ID_UNIQUE #elif defined(__WINPTHREADS_VERSION_MAJOR) #define NUMERIC_THREAD_ID(id)((unsigned long)(id)) #define THREAD_EQUAL(id1,id2)((id1)==(id2)) #ifndef _WIN64 #define NUMERIC_THREAD_ID_UNIQUE #endif #else #define NUMERIC_THREAD_ID(id)((unsigned long)(word)(id.p)) #define THREAD_EQUAL(id1,id2)((id1.p==id2.p)&&(id1.x==id2.x)) #undef NUMERIC_THREAD_ID_UNIQUE #endif #define NO_THREAD ((unsigned long)(-1l)) #ifdef SN_TARGET_PSP2 EXTERN_C_END #include "psp2-support.h" EXTERN_C_BEGIN GC_EXTERN WapiMutex GC_allocate_ml_PSP2; #define UNCOND_LOCK(){ int res;GC_ASSERT(I_DONT_HOLD_LOCK());res=PSP2_MutexLock(&GC_allocate_ml_PSP2);GC_ASSERT(0==res);(void)res;SET_LOCK_HOLDER();} #define UNCOND_UNLOCK(){ int res;GC_ASSERT(I_HOLD_LOCK());UNSET_LOCK_HOLDER();res=PSP2_MutexUnlock(&GC_allocate_ml_PSP2);GC_ASSERT(0==res);(void)res;} #elif (!defined(THREAD_LOCAL_ALLOC)||defined(USE_SPIN_LOCK))&&!defined(USE_PTHREAD_LOCKS) #undef USE_SPIN_LOCK #define USE_SPIN_LOCK GC_EXTERN volatile AO_TS_t GC_allocate_lock; GC_INNER void GC_lock(void); #ifdef GC_ASSERTIONS #define UNCOND_LOCK(){ GC_ASSERT(I_DONT_HOLD_LOCK());if (AO_test_and_set_acquire(&GC_allocate_lock)==AO_TS_SET)GC_lock();SET_LOCK_HOLDER();} #define UNCOND_UNLOCK(){ GC_ASSERT(I_HOLD_LOCK());UNSET_LOCK_HOLDER();AO_CLEAR(&GC_allocate_lock);} #else #define UNCOND_LOCK(){ if (AO_test_and_set_acquire(&GC_allocate_lock)==AO_TS_SET)GC_lock();} #define UNCOND_UNLOCK()AO_CLEAR(&GC_allocate_lock) #endif #else #ifndef USE_PTHREAD_LOCKS #define USE_PTHREAD_LOCKS #endif #endif #ifdef USE_PTHREAD_LOCKS EXTERN_C_END #include EXTERN_C_BEGIN GC_EXTERN pthread_mutex_t GC_allocate_ml; #ifdef GC_ASSERTIONS GC_INNER void GC_lock(void); #define UNCOND_LOCK(){ GC_ASSERT(I_DONT_HOLD_LOCK());GC_lock();SET_LOCK_HOLDER();} #define UNCOND_UNLOCK(){ GC_ASSERT(I_HOLD_LOCK());UNSET_LOCK_HOLDER();pthread_mutex_unlock(&GC_allocate_ml);} #else #if defined(NO_PTHREAD_TRYLOCK) #define UNCOND_LOCK()pthread_mutex_lock(&GC_allocate_ml) #else GC_INNER void GC_lock(void); #define UNCOND_LOCK(){ if (0!=pthread_mutex_trylock(&GC_allocate_ml))GC_lock();} #endif #define UNCOND_UNLOCK()pthread_mutex_unlock(&GC_allocate_ml) #endif #endif #ifdef GC_ASSERTIONS GC_EXTERN unsigned long GC_lock_holder; #define SET_LOCK_HOLDER()GC_lock_holder=NUMERIC_THREAD_ID(pthread_self()) #define UNSET_LOCK_HOLDER()GC_lock_holder=NO_THREAD #define I_HOLD_LOCK()(!GC_need_to_lock||GC_lock_holder==NUMERIC_THREAD_ID(pthread_self())) #if!defined(NUMERIC_THREAD_ID_UNIQUE)||defined(THREAD_SANITIZER) #define I_DONT_HOLD_LOCK()TRUE #else #define I_DONT_HOLD_LOCK()(!GC_need_to_lock||GC_lock_holder!=NUMERIC_THREAD_ID(pthread_self())) #endif #endif #ifndef GC_WIN32_THREADS GC_EXTERN volatile GC_bool GC_collecting; #ifdef AO_HAVE_char_store #define ENTER_GC()AO_char_store((unsigned char*)&GC_collecting,TRUE) #define EXIT_GC()AO_char_store((unsigned char*)&GC_collecting,FALSE) #else #define ENTER_GC()(void)(GC_collecting=TRUE) #define EXIT_GC()(void)(GC_collecting=FALSE) #endif #endif #endif #if defined(GC_ALWAYS_MULTITHREADED)&&(defined(USE_PTHREAD_LOCKS)||defined(USE_SPIN_LOCK)) #define GC_need_to_lock TRUE #define set_need_to_lock()(void)0 #else #if defined(GC_ALWAYS_MULTITHREADED)&&!defined(CPPCHECK) #error Runtime initialization of GC lock is needed! #endif #undef GC_ALWAYS_MULTITHREADED GC_EXTERN GC_bool GC_need_to_lock; #ifdef THREAD_SANITIZER #define set_need_to_lock()(void)(*(GC_bool volatile*)&GC_need_to_lock?FALSE:(GC_need_to_lock=TRUE)) #else #define set_need_to_lock()(void)(GC_need_to_lock=TRUE) #endif #endif EXTERN_C_END #else #define LOCK()(void)0 #define UNLOCK()(void)0 #ifdef GC_ASSERTIONS #define I_HOLD_LOCK()TRUE #define I_DONT_HOLD_LOCK()TRUE #endif #endif #if defined(UNCOND_LOCK)&&!defined(LOCK) #if (defined(LINT2)&&defined(USE_PTHREAD_LOCKS))||defined(GC_ALWAYS_MULTITHREADED) #define LOCK()UNCOND_LOCK() #define UNLOCK()UNCOND_UNLOCK() #else #define LOCK()do { if (GC_need_to_lock)UNCOND_LOCK();} while (0) #define UNLOCK()do { if (GC_need_to_lock)UNCOND_UNLOCK();} while (0) #endif #endif #ifndef ENTER_GC #define ENTER_GC() #define EXIT_GC() #endif #ifndef DCL_LOCK_STATE #define DCL_LOCK_STATE #endif #endif #define GC_WORD_MAX (~(word)0) #ifdef STACK_GROWS_DOWN #define COOLER_THAN > #define HOTTER_THAN < #define MAKE_COOLER(x,y)if ((word)((x)+(y))> (word)(x)){(x)+=(y);} else (x)=(ptr_t)GC_WORD_MAX #define MAKE_HOTTER(x,y)(x)-=(y) #else #define COOLER_THAN < #define HOTTER_THAN > #define MAKE_COOLER(x,y)if ((word)((x)- (y))< (word)(x)){(x)-=(y);} else (x)=0 #define MAKE_HOTTER(x,y)(x)+=(y) #endif #if defined(AMIGA)&&defined(__SASC) #define GC_FAR __far #else #define GC_FAR #endif EXTERN_C_BEGIN #ifndef GC_NO_FINALIZATION #define GC_INVOKE_FINALIZERS()GC_notify_or_invoke_finalizers() GC_INNER void GC_notify_or_invoke_finalizers(void); GC_INNER void GC_finalize(void); #ifndef GC_TOGGLE_REFS_NOT_NEEDED GC_INNER void GC_process_togglerefs(void); #endif #ifndef SMALL_CONFIG GC_INNER void GC_print_finalization_stats(void); #endif #else #define GC_INVOKE_FINALIZERS()(void)0 #endif #if!defined(DONT_ADD_BYTE_AT_END) #ifdef LINT2 #define EXTRA_BYTES ((size_t)(GC_all_interior_pointers?1:0)) #else #define EXTRA_BYTES (size_t)GC_all_interior_pointers #endif #define MAX_EXTRA_BYTES 1 #else #define EXTRA_BYTES 0 #define MAX_EXTRA_BYTES 0 #endif #ifndef LARGE_CONFIG #define MINHINCR 16 #define MAXHINCR 2048 #else #define MINHINCR 64 #define MAXHINCR 4096 #endif #define BL_LIMIT GC_black_list_spacing #ifdef NEED_CALLINFO struct callinfo { word ci_pc; #if NARGS > 0 word ci_arg[NARGS]; #endif #if (NFRAMES*(NARGS+1))% 2==1 word ci_dummy; #endif }; #endif #ifdef SAVE_CALL_CHAIN GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]); GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]); #endif EXTERN_C_END #ifndef NO_CLOCK #ifdef BSD_TIME #undef CLOCK_TYPE #undef GET_TIME #undef MS_TIME_DIFF #define CLOCK_TYPE struct timeval #define CLOCK_TYPE_INITIALIZER { 0,0 } #define GET_TIME(x)do { struct rusage rusage;getrusage(RUSAGE_SELF,&rusage);x=rusage.ru_utime;} while (0) #define MS_TIME_DIFF(a,b)((unsigned long)((long)(a.tv_sec-b.tv_sec)*1000+(long)(a.tv_usec - b.tv_usec)/1000 - (a.tv_usec < b.tv_usec&&(long)(a.tv_usec - b.tv_usec)% 1000!=0?1:0))) #define NS_FRAC_TIME_DIFF(a,b)((unsigned long)((a.tv_usec < b.tv_usec&&(long)(a.tv_usec - b.tv_usec)% 1000!=0?1000L:0)+(long)(a.tv_usec - b.tv_usec)% 1000)*1000) #elif defined(MSWIN32)||defined(MSWINCE)||defined(WINXP_USE_PERF_COUNTER) #if defined(MSWINRT_FLAVOR)||defined(WINXP_USE_PERF_COUNTER) #define CLOCK_TYPE ULONGLONG #define GET_TIME(x)do { LARGE_INTEGER freq,tc;if (!QueryPerformanceFrequency(&freq)||!QueryPerformanceCounter(&tc))ABORT("QueryPerformanceCounter requires WinXP+");x=(CLOCK_TYPE)((double)tc.QuadPart/freq.QuadPart*1e9);} while (0) #define MS_TIME_DIFF(a,b)((unsigned long)(((a)- (b))/1000000UL)) #define NS_FRAC_TIME_DIFF(a,b)((unsigned long)(((a)- (b))% 1000000UL)) #else #define CLOCK_TYPE DWORD #define GET_TIME(x)(void)(x=GetTickCount()) #define MS_TIME_DIFF(a,b)((unsigned long)((a)- (b))) #define NS_FRAC_TIME_DIFF(a,b)0UL #endif #elif defined(NN_PLATFORM_CTR) #define CLOCK_TYPE long long EXTERN_C_BEGIN CLOCK_TYPE n3ds_get_system_tick(void); CLOCK_TYPE n3ds_convert_tick_to_ms(CLOCK_TYPE tick); EXTERN_C_END #define GET_TIME(x)(void)(x=n3ds_get_system_tick()) #define MS_TIME_DIFF(a,b)((unsigned long)n3ds_convert_tick_to_ms((a)-(b))) #define NS_FRAC_TIME_DIFF(a,b)0UL #elif defined(NINTENDO_SWITCH)||(((defined(LINUX)&&defined(__USE_POSIX199309))||defined(CYGWIN32))&&defined(_POSIX_TIMERS)) #include #define CLOCK_TYPE struct timespec #define CLOCK_TYPE_INITIALIZER { 0,0 } #if defined(_POSIX_MONOTONIC_CLOCK)&&!defined(NINTENDO_SWITCH) #define GET_TIME(x)do { if (clock_gettime(CLOCK_MONOTONIC,&x)==-1)ABORT("clock_gettime failed");} while (0) #else #define GET_TIME(x)do { if (clock_gettime(CLOCK_REALTIME,&x)==-1)ABORT("clock_gettime failed");} while (0) #endif #define MS_TIME_DIFF(a,b)((unsigned long)((a).tv_nsec+(1000000L*1000 - (b).tv_nsec))/1000000UL+((unsigned long)((a).tv_sec - (b).tv_sec)*1000UL)- 1000UL) #define NS_FRAC_TIME_DIFF(a,b)((unsigned long)((a).tv_nsec+(1000000L*1000 - (b).tv_nsec))% 1000000UL) #else #include #if defined(FREEBSD)&&!defined(CLOCKS_PER_SEC) #include #define CLOCKS_PER_SEC CLK_TCK #endif #if!defined(CLOCKS_PER_SEC) #define CLOCKS_PER_SEC 1000000 #endif #define CLOCK_TYPE clock_t #define GET_TIME(x)(void)(x=clock()) #define MS_TIME_DIFF(a,b)(CLOCKS_PER_SEC % 1000==0?(unsigned long)((a)- (b))/(unsigned long)(CLOCKS_PER_SEC/1000):((unsigned long)((a)- (b))*1000)/(unsigned long)CLOCKS_PER_SEC) #define NS_FRAC_TIME_DIFF(a,b)(CLOCKS_PER_SEC<=1000?0UL:(unsigned long)(CLOCKS_PER_SEC<=(clock_t)1000000UL?(((a)- (b))*((clock_t)1000000UL/CLOCKS_PER_SEC)% 1000)*1000:(CLOCKS_PER_SEC<=(clock_t)1000000UL*1000?((a)- (b))*((clock_t)1000000UL*1000/CLOCKS_PER_SEC):(((a)- (b))*(clock_t)1000000UL*1000)/CLOCKS_PER_SEC)% (clock_t)1000000UL)) #endif #ifndef CLOCK_TYPE_INITIALIZER #define CLOCK_TYPE_INITIALIZER 0 #endif #endif #if defined(SPARC)&&defined(SUNOS4)||(defined(M68K)&&defined(NEXT))||defined(VAX) #define BCOPY_EXISTS #elif defined(AMIGA)||defined(DARWIN) #include #define BCOPY_EXISTS #elif defined(MACOS)&&defined(POWERPC) #include #define bcopy(x,y,n)BlockMoveData(x,y,n) #define bzero(x,n)BlockZero(x,n) #define BCOPY_EXISTS #endif #if!defined(BCOPY_EXISTS)||defined(CPPCHECK) #include #define BCOPY(x,y,n)memcpy(y,x,(size_t)(n)) #define BZERO(x,n)memset(x,0,(size_t)(n)) #else #define BCOPY(x,y,n)bcopy((void*)(x),(void*)(y),(size_t)(n)) #define BZERO(x,n)bzero((void*)(x),(size_t)(n)) #endif #ifdef PCR #include "th/PCR_ThCtl.h" #endif EXTERN_C_BEGIN #ifdef PCR #define STOP_WORLD()PCR_ThCtl_SetExclusiveMode(PCR_ThCtl_ExclusiveMode_stopNormal,PCR_allSigsBlocked,PCR_waitForever) #define START_WORLD()PCR_ThCtl_SetExclusiveMode(PCR_ThCtl_ExclusiveMode_null,PCR_allSigsBlocked,PCR_waitForever) #else #if defined(NN_PLATFORM_CTR)||defined(NINTENDO_SWITCH)||defined(GC_WIN32_THREADS)||defined(GC_PTHREADS) GC_INNER void GC_stop_world(void); GC_INNER void GC_start_world(void); #define STOP_WORLD()GC_stop_world() #define START_WORLD()GC_start_world() #else #define STOP_WORLD()GC_ASSERT(GC_blocked_sp==NULL) #define START_WORLD() #endif #endif #ifdef THREADS GC_EXTERN GC_on_thread_event_proc GC_on_thread_event; #endif #if defined(SMALL_CONFIG)||defined(PCR) #define GC_on_abort(msg)(void)0 #else GC_API_PRIV GC_abort_func GC_on_abort; #endif #if defined(CPPCHECK) #define ABORT(msg){ GC_on_abort(msg);abort();} #elif defined(PCR) #define ABORT(s)PCR_Base_Panic(s) #else #if defined(MSWIN_XBOX1)&&!defined(DebugBreak) #define DebugBreak()__debugbreak() #elif defined(MSWINCE)&&!defined(DebugBreak)&&(!defined(UNDER_CE)||(defined(__MINGW32CE__)&&!defined(ARM32))) #define DebugBreak()_exit(-1) #endif #if defined(MSWIN32)&&(defined(NO_DEBUGGING)||defined(LINT2)) #define ABORT(msg)(GC_on_abort(msg),_exit(-1)) #elif defined(MSWINCE)&&defined(NO_DEBUGGING) #define ABORT(msg)(GC_on_abort(msg),ExitProcess(-1)) #elif defined(MSWIN32)||defined(MSWINCE) #if defined(_CrtDbgBreak)&&defined(_DEBUG)&&defined(_MSC_VER) #define ABORT(msg){ GC_on_abort(msg);_CrtDbgBreak();} #else #define ABORT(msg){ GC_on_abort(msg);DebugBreak();} #endif #else #define ABORT(msg)(GC_on_abort(msg),abort()) #endif #endif #define ABORT_ARG1(C_msg,C_fmt,arg1)MACRO_BLKSTMT_BEGIN GC_INFOLOG_PRINTF(C_msg C_fmt "\n",arg1);ABORT(C_msg);MACRO_BLKSTMT_END #define ABORT_ARG2(C_msg,C_fmt,arg1,arg2)MACRO_BLKSTMT_BEGIN GC_INFOLOG_PRINTF(C_msg C_fmt "\n",arg1,arg2);ABORT(C_msg);MACRO_BLKSTMT_END #define ABORT_ARG3(C_msg,C_fmt,arg1,arg2,arg3)MACRO_BLKSTMT_BEGIN GC_INFOLOG_PRINTF(C_msg C_fmt "\n",arg1,arg2,arg3);ABORT(C_msg);MACRO_BLKSTMT_END #define ABORT_RET(msg)if ((signed_word)GC_current_warn_proc==-1){} else ABORT(msg) #ifdef PCR #define EXIT()PCR_Base_Exit(1,PCR_waitForever) #else #define EXIT()(GC_on_abort(NULL),exit(1)) #endif #define WARN(msg,arg)(*GC_current_warn_proc)(( char*)("GC Warning:" msg),(word)(arg)) GC_EXTERN GC_warn_proc GC_current_warn_proc; #ifndef WARN_PRIdPTR #define WARN_PRIdPTR "ld" #endif #define TRUSTED_STRING(s)(char*)COVERT_DATAFLOW(s) #ifdef GC_READ_ENV_FILE GC_INNER char*GC_envfile_getenv(const char*name); #define GETENV(name)GC_envfile_getenv(name) #elif defined(NO_GETENV)&&!defined(CPPCHECK) #define GETENV(name)NULL #elif defined(EMPTY_GETENV_RESULTS) GC_INLINE char*fixed_getenv(const char*name) { char*value=getenv(name); return value!=NULL&&*value!='\0'?value:NULL; } #define GETENV(name)fixed_getenv(name) #else #define GETENV(name)getenv(name) #endif EXTERN_C_END #if defined(DARWIN) #include #ifndef MAC_OS_X_VERSION_MAX_ALLOWED #include #endif #if defined(POWERPC) #if CPP_WORDSZ==32 #define GC_THREAD_STATE_T ppc_thread_state_t #else #define GC_THREAD_STATE_T ppc_thread_state64_t #define GC_MACH_THREAD_STATE PPC_THREAD_STATE64 #define GC_MACH_THREAD_STATE_COUNT PPC_THREAD_STATE64_COUNT #endif #elif defined(I386)||defined(X86_64) #if CPP_WORDSZ==32 #if defined(i386_THREAD_STATE_COUNT)&&!defined(x86_THREAD_STATE32_COUNT) #define GC_THREAD_STATE_T i386_thread_state_t #define GC_MACH_THREAD_STATE i386_THREAD_STATE #define GC_MACH_THREAD_STATE_COUNT i386_THREAD_STATE_COUNT #else #define GC_THREAD_STATE_T x86_thread_state32_t #define GC_MACH_THREAD_STATE x86_THREAD_STATE32 #define GC_MACH_THREAD_STATE_COUNT x86_THREAD_STATE32_COUNT #endif #else #define GC_THREAD_STATE_T x86_thread_state64_t #define GC_MACH_THREAD_STATE x86_THREAD_STATE64 #define GC_MACH_THREAD_STATE_COUNT x86_THREAD_STATE64_COUNT #endif #elif defined(ARM32)&&defined(ARM_UNIFIED_THREAD_STATE)&&!defined(CPPCHECK) #define GC_THREAD_STATE_T arm_unified_thread_state_t #define GC_MACH_THREAD_STATE ARM_UNIFIED_THREAD_STATE #define GC_MACH_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT #elif defined(ARM32) #define GC_THREAD_STATE_T arm_thread_state_t #ifdef ARM_MACHINE_THREAD_STATE_COUNT #define GC_MACH_THREAD_STATE ARM_MACHINE_THREAD_STATE #define GC_MACH_THREAD_STATE_COUNT ARM_MACHINE_THREAD_STATE_COUNT #endif #elif defined(AARCH64) #define GC_THREAD_STATE_T arm_thread_state64_t #define GC_MACH_THREAD_STATE ARM_THREAD_STATE64 #define GC_MACH_THREAD_STATE_COUNT ARM_THREAD_STATE64_COUNT #elif!defined(CPPCHECK) #error define GC_THREAD_STATE_T #endif #ifndef GC_MACH_THREAD_STATE #define GC_MACH_THREAD_STATE MACHINE_THREAD_STATE #define GC_MACH_THREAD_STATE_COUNT MACHINE_THREAD_STATE_COUNT #endif #if CPP_WORDSZ==32 #define GC_MACH_HEADER mach_header #define GC_MACH_SECTION section #define GC_GETSECTBYNAME getsectbynamefromheader #else #define GC_MACH_HEADER mach_header_64 #define GC_MACH_SECTION section_64 #define GC_GETSECTBYNAME getsectbynamefromheader_64 #endif #if __DARWIN_UNIX03 #define THREAD_FLD_NAME(x)__ ## x #else #define THREAD_FLD_NAME(x)x #endif #if defined(ARM32)&&defined(ARM_UNIFIED_THREAD_STATE) #define THREAD_FLD(x)ts_32.THREAD_FLD_NAME(x) #else #define THREAD_FLD(x)THREAD_FLD_NAME(x) #endif #endif #include #if __STDC_VERSION__>=201112L #include #endif EXTERN_C_BEGIN #if CPP_WORDSZ==32 #define WORDS_TO_BYTES(x)((x)<<2) #define BYTES_TO_WORDS(x)((x)>>2) #define LOGWL ((word)5) #define modWORDSZ(n)((n)&0x1f) #if ALIGNMENT!=4 #define UNALIGNED_PTRS #endif #endif #if CPP_WORDSZ==64 #define WORDS_TO_BYTES(x)((x)<<3) #define BYTES_TO_WORDS(x)((x)>>3) #define LOGWL ((word)6) #define modWORDSZ(n)((n)&0x3f) #if ALIGNMENT!=8 #define UNALIGNED_PTRS #endif #endif #define GRANULE_BYTES GC_GRANULE_BYTES #define TINY_FREELISTS GC_TINY_FREELISTS #define WORDSZ ((word)CPP_WORDSZ) #define SIGNB ((word)1<<(WORDSZ-1)) #define BYTES_PER_WORD ((word)(sizeof (word))) #define divWORDSZ(n)((n)>>LOGWL) #if GRANULE_BYTES==8 #define BYTES_TO_GRANULES(n)((n)>>3) #define GRANULES_TO_BYTES(n)((n)<<3) #if CPP_WORDSZ==64 #define GRANULES_TO_WORDS(n)(n) #elif CPP_WORDSZ==32 #define GRANULES_TO_WORDS(n)((n)<<1) #else #define GRANULES_TO_WORDS(n)BYTES_TO_WORDS(GRANULES_TO_BYTES(n)) #endif #elif GRANULE_BYTES==16 #define BYTES_TO_GRANULES(n)((n)>>4) #define GRANULES_TO_BYTES(n)((n)<<4) #if CPP_WORDSZ==64 #define GRANULES_TO_WORDS(n)((n)<<1) #elif CPP_WORDSZ==32 #define GRANULES_TO_WORDS(n)((n)<<2) #else #define GRANULES_TO_WORDS(n)BYTES_TO_WORDS(GRANULES_TO_BYTES(n)) #endif #else #error Bad GRANULE_BYTES value #endif #ifndef HBLKSIZE #if defined(LARGE_CONFIG)||!defined(SMALL_CONFIG) #ifdef ALPHA #define CPP_LOG_HBLKSIZE 13 #elif defined(SN_TARGET_ORBIS)||defined(SN_TARGET_PSP2) #define CPP_LOG_HBLKSIZE 16 #else #define CPP_LOG_HBLKSIZE 12 #endif #else #define CPP_LOG_HBLKSIZE 10 #endif #else #if HBLKSIZE==512 #define CPP_LOG_HBLKSIZE 9 #elif HBLKSIZE==1024 #define CPP_LOG_HBLKSIZE 10 #elif HBLKSIZE==2048 #define CPP_LOG_HBLKSIZE 11 #elif HBLKSIZE==4096 #define CPP_LOG_HBLKSIZE 12 #elif HBLKSIZE==8192 #define CPP_LOG_HBLKSIZE 13 #elif HBLKSIZE==16384 #define CPP_LOG_HBLKSIZE 14 #elif!defined(CPPCHECK) #error Bad HBLKSIZE value #endif #undef HBLKSIZE #endif #define CPP_HBLKSIZE (1<>LOG_HBLKSIZE) #define HBLK_PTR_DIFF(p,q)divHBLKSZ((ptr_t)p - (ptr_t)q) #define modHBLKSZ(n)((n)&(HBLKSIZE-1)) #define HBLKPTR(objptr)((struct hblk*)(((word)(objptr))&~(word)(HBLKSIZE-1))) #define HBLKDISPL(objptr)(((size_t)(objptr))&(HBLKSIZE-1)) #define ROUNDUP_GRANULE_SIZE(lb)(SIZET_SAT_ADD(lb,GRANULE_BYTES - 1)&~(GRANULE_BYTES - 1)) #define ROUNDED_UP_GRANULES(lb)BYTES_TO_GRANULES(SIZET_SAT_ADD(lb,GRANULE_BYTES - 1+EXTRA_BYTES)) #if MAX_EXTRA_BYTES==0 #define SMALL_OBJ(bytes)EXPECT((bytes)<=(MAXOBJBYTES),TRUE) #else #define SMALL_OBJ(bytes)(EXPECT((bytes)<=(MAXOBJBYTES - MAX_EXTRA_BYTES),TRUE)||(bytes)<=MAXOBJBYTES - EXTRA_BYTES) #endif #define ADD_SLOP(lb)SIZET_SAT_ADD(lb,EXTRA_BYTES) #ifdef LARGE_CONFIG #if CPP_WORDSZ==32 #define LOG_PHT_ENTRIES 20 #else #define LOG_PHT_ENTRIES 21 #endif #elif!defined(SMALL_CONFIG) #define LOG_PHT_ENTRIES 18 #else #define LOG_PHT_ENTRIES 15 #endif #define PHT_ENTRIES ((word)1<>LOGWL) typedef word page_hash_table[PHT_SIZE]; #define PHT_HASH(addr)((((word)(addr))>>LOG_HBLKSIZE)&(PHT_ENTRIES - 1)) #define get_pht_entry_from_index(bl,index)(((bl)[divWORDSZ(index)]>>modWORDSZ(index))&1) #define set_pht_entry_from_index(bl,index)(void)((bl)[divWORDSZ(index)]|=(word)1<hb_flags&FREE_BLK)!=0) #define OBJ_SZ_TO_BLOCKS(lb)divHBLKSZ((lb)+HBLKSIZE-1) #define OBJ_SZ_TO_BLOCKS_CHECKED(lb)divHBLKSZ(SIZET_SAT_ADD(lb,HBLKSIZE - 1)) #define obj_link(p)(*(void**)(p)) #define LOG_MAX_MARK_PROCS 6 #define MAX_MARK_PROCS (1< 32 #define MAX_HEAP_SECTS 81920 #else #define MAX_HEAP_SECTS 7680 #endif #elif defined(SMALL_CONFIG)&&!defined(USE_PROC_FOR_LIBRARIES) #if defined(PARALLEL_MARK)&&(defined(MSWIN32)||defined(CYGWIN32)) #define MAX_HEAP_SECTS 384 #else #define MAX_HEAP_SECTS 128 #endif #elif CPP_WORDSZ > 32 #define MAX_HEAP_SECTS 1024 #else #define MAX_HEAP_SECTS 512 #endif #endif typedef struct GC_ms_entry { ptr_t mse_start; union word_ptr_ao_u mse_descr; } mse; typedef int mark_state_t; struct disappearing_link; struct finalizable_object; struct dl_hashtbl_s { struct disappearing_link**head; word entries; unsigned log_size; }; struct fnlz_roots_s { struct finalizable_object**fo_head; struct finalizable_object*finalize_now; }; union toggle_ref_u { void*strong_ref; GC_hidden_pointer weak_ref; }; typedef struct { word ed_bitmap; GC_bool ed_continued; } typed_ext_descr_t; struct _GC_arrays { word _heapsize; word _requested_heapsize; ptr_t _last_heap_addr; ptr_t _prev_heap_addr; word _large_free_bytes; word _large_allocd_bytes; word _max_large_allocd_bytes; word _bytes_allocd_before_gc; #ifndef SEPARATE_GLOBALS #define GC_bytes_allocd GC_arrays._bytes_allocd word _bytes_allocd; #endif word _bytes_dropped; word _bytes_finalized; word _bytes_freed; word _finalizer_bytes_freed; bottom_index*_all_bottom_indices; bottom_index*_all_bottom_indices_end; ptr_t _scratch_free_ptr; hdr*_hdr_free_list; ptr_t _scratch_end_ptr; ptr_t _scratch_last_end_ptr; mse*_mark_stack; mse*_mark_stack_limit; #ifdef PARALLEL_MARK mse*volatile _mark_stack_top; #else mse*_mark_stack_top; #endif word _composite_in_use; word _atomic_in_use; #ifdef USE_MUNMAP #define GC_unmapped_bytes GC_arrays._unmapped_bytes word _unmapped_bytes; #ifdef COUNT_UNMAPPED_REGIONS #define GC_num_unmapped_regions GC_arrays._num_unmapped_regions signed_word _num_unmapped_regions; #endif #else #define GC_unmapped_bytes 0 #endif bottom_index*_all_nils; #define GC_scan_ptr GC_arrays._scan_ptr struct hblk*_scan_ptr; #ifdef PARALLEL_MARK #define GC_main_local_mark_stack GC_arrays._main_local_mark_stack mse*_main_local_mark_stack; #define GC_first_nonempty GC_arrays._first_nonempty volatile AO_t _first_nonempty; #endif #define GC_mark_stack_size GC_arrays._mark_stack_size size_t _mark_stack_size; #define GC_mark_state GC_arrays._mark_state mark_state_t _mark_state; #define GC_mark_stack_too_small GC_arrays._mark_stack_too_small GC_bool _mark_stack_too_small; #define GC_objects_are_marked GC_arrays._objects_are_marked GC_bool _objects_are_marked; #ifdef ENABLE_TRACE #define GC_trace_addr GC_arrays._trace_addr ptr_t _trace_addr; #endif #define GC_n_heap_sects GC_arrays._n_heap_sects word _n_heap_sects; #if defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32) #define GC_n_heap_bases GC_arrays._n_heap_bases word _n_heap_bases; #endif #ifdef USE_PROC_FOR_LIBRARIES #define GC_n_memory GC_arrays._n_memory word _n_memory; #endif #ifdef GC_GCJ_SUPPORT #define GC_gcjobjfreelist GC_arrays._gcjobjfreelist ptr_t*_gcjobjfreelist; #endif #define GC_fo_entries GC_arrays._fo_entries word _fo_entries; #ifndef GC_NO_FINALIZATION #define GC_dl_hashtbl GC_arrays._dl_hashtbl #define GC_fnlz_roots GC_arrays._fnlz_roots #define GC_log_fo_table_size GC_arrays._log_fo_table_size #ifndef GC_LONG_REFS_NOT_NEEDED #define GC_ll_hashtbl GC_arrays._ll_hashtbl struct dl_hashtbl_s _ll_hashtbl; #endif struct dl_hashtbl_s _dl_hashtbl; struct fnlz_roots_s _fnlz_roots; unsigned _log_fo_table_size; #ifndef GC_TOGGLE_REFS_NOT_NEEDED #define GC_toggleref_arr GC_arrays._toggleref_arr #define GC_toggleref_array_size GC_arrays._toggleref_array_size #define GC_toggleref_array_capacity GC_arrays._toggleref_array_capacity union toggle_ref_u*_toggleref_arr; size_t _toggleref_array_size; size_t _toggleref_array_capacity; #endif #endif #ifdef TRACE_BUF #define GC_trace_buf_ptr GC_arrays._trace_buf_ptr int _trace_buf_ptr; #endif #ifdef ENABLE_DISCLAIM #define GC_finalized_kind GC_arrays._finalized_kind int _finalized_kind; #endif #define n_root_sets GC_arrays._n_root_sets #define GC_excl_table_entries GC_arrays._excl_table_entries int _n_root_sets; size_t _excl_table_entries; #ifdef THREADS #define GC_roots_were_cleared GC_arrays._roots_were_cleared GC_bool _roots_were_cleared; #endif #define GC_explicit_typing_initialized GC_arrays._explicit_typing_initialized #define GC_ed_size GC_arrays._ed_size #define GC_avail_descr GC_arrays._avail_descr #define GC_ext_descriptors GC_arrays._ext_descriptors #ifdef AO_HAVE_load_acquire volatile AO_t _explicit_typing_initialized; #else GC_bool _explicit_typing_initialized; #endif size_t _ed_size; size_t _avail_descr; typed_ext_descr_t*_ext_descriptors; GC_mark_proc _mark_procs[MAX_MARK_PROCS]; char _modws_valid_offsets[sizeof(word)]; #if!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(CYGWIN32) #define GC_root_index GC_arrays._root_index struct roots*_root_index[RT_SIZE]; #endif #ifdef SAVE_CALL_CHAIN #define GC_last_stack GC_arrays._last_stack struct callinfo _last_stack[NFRAMES]; #endif #ifndef SEPARATE_GLOBALS #define GC_objfreelist GC_arrays._objfreelist void*_objfreelist[MAXOBJGRANULES+1]; #define GC_aobjfreelist GC_arrays._aobjfreelist void*_aobjfreelist[MAXOBJGRANULES+1]; #endif void*_uobjfreelist[MAXOBJGRANULES+1]; #ifdef GC_ATOMIC_UNCOLLECTABLE #define GC_auobjfreelist GC_arrays._auobjfreelist void*_auobjfreelist[MAXOBJGRANULES+1]; #endif size_t _size_map[MAXOBJBYTES+1]; #ifdef MARK_BIT_PER_GRANULE #define GC_obj_map GC_arrays._obj_map unsigned short*_obj_map[MAXOBJGRANULES+1]; #define MAP_LEN BYTES_TO_GRANULES(HBLKSIZE) #endif #define VALID_OFFSET_SZ HBLKSIZE char _valid_offsets[VALID_OFFSET_SZ]; #ifndef GC_DISABLE_INCREMENTAL #define GC_grungy_pages GC_arrays._grungy_pages page_hash_table _grungy_pages; #define GC_dirty_pages GC_arrays._dirty_pages volatile page_hash_table _dirty_pages; #endif #if (defined(CHECKSUMS)&&defined(GWW_VDB))||defined(PROC_VDB) #define GC_written_pages GC_arrays._written_pages page_hash_table _written_pages; #endif #define GC_heap_sects GC_arrays._heap_sects struct HeapSect { ptr_t hs_start; size_t hs_bytes; } _heap_sects[MAX_HEAP_SECTS]; #if defined(USE_PROC_FOR_LIBRARIES) #define GC_our_memory GC_arrays._our_memory struct HeapSect _our_memory[MAX_HEAP_SECTS]; #endif #if defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32) #define GC_heap_bases GC_arrays._heap_bases ptr_t _heap_bases[MAX_HEAP_SECTS]; #endif #ifdef MSWINCE #define GC_heap_lengths GC_arrays._heap_lengths word _heap_lengths[MAX_HEAP_SECTS]; #endif struct roots _static_roots[MAX_ROOT_SETS]; struct exclusion _excl_table[MAX_EXCLUSIONS]; bottom_index*_top_index[TOP_SZ]; }; GC_API_PRIV GC_FAR struct _GC_arrays GC_arrays; #define GC_all_nils GC_arrays._all_nils #define GC_atomic_in_use GC_arrays._atomic_in_use #define GC_bytes_allocd_before_gc GC_arrays._bytes_allocd_before_gc #define GC_bytes_dropped GC_arrays._bytes_dropped #define GC_bytes_finalized GC_arrays._bytes_finalized #define GC_bytes_freed GC_arrays._bytes_freed #define GC_composite_in_use GC_arrays._composite_in_use #define GC_excl_table GC_arrays._excl_table #define GC_finalizer_bytes_freed GC_arrays._finalizer_bytes_freed #define GC_heapsize GC_arrays._heapsize #define GC_large_allocd_bytes GC_arrays._large_allocd_bytes #define GC_large_free_bytes GC_arrays._large_free_bytes #define GC_last_heap_addr GC_arrays._last_heap_addr #define GC_mark_stack GC_arrays._mark_stack #define GC_mark_stack_limit GC_arrays._mark_stack_limit #define GC_mark_stack_top GC_arrays._mark_stack_top #define GC_mark_procs GC_arrays._mark_procs #define GC_max_large_allocd_bytes GC_arrays._max_large_allocd_bytes #define GC_modws_valid_offsets GC_arrays._modws_valid_offsets #define GC_prev_heap_addr GC_arrays._prev_heap_addr #define GC_requested_heapsize GC_arrays._requested_heapsize #define GC_all_bottom_indices GC_arrays._all_bottom_indices #define GC_all_bottom_indices_end GC_arrays._all_bottom_indices_end #define GC_scratch_free_ptr GC_arrays._scratch_free_ptr #define GC_hdr_free_list GC_arrays._hdr_free_list #define GC_scratch_end_ptr GC_arrays._scratch_end_ptr #define GC_scratch_last_end_ptr GC_arrays._scratch_last_end_ptr #define GC_size_map GC_arrays._size_map #define GC_static_roots GC_arrays._static_roots #define GC_top_index GC_arrays._top_index #define GC_uobjfreelist GC_arrays._uobjfreelist #define GC_valid_offsets GC_arrays._valid_offsets #define beginGC_arrays ((ptr_t)(&GC_arrays)) #define endGC_arrays (((ptr_t)(&GC_arrays))+(sizeof GC_arrays)) #define USED_HEAP_SIZE (GC_heapsize - GC_large_free_bytes) #ifndef MAXOBJKINDS #define MAXOBJKINDS 16 #endif GC_EXTERN struct obj_kind { void**ok_freelist; struct hblk**ok_reclaim_list; word ok_descriptor; GC_bool ok_relocate_descr; GC_bool ok_init; #ifdef ENABLE_DISCLAIM GC_bool ok_mark_unconditionally; int (GC_CALLBACK*ok_disclaim_proc)(void*); #define OK_DISCLAIM_INITZ,FALSE,0 #else #define OK_DISCLAIM_INITZ #endif } GC_obj_kinds[MAXOBJKINDS]; #define beginGC_obj_kinds ((ptr_t)(&GC_obj_kinds)) #define endGC_obj_kinds (beginGC_obj_kinds+(sizeof GC_obj_kinds)) #ifdef SEPARATE_GLOBALS extern word GC_bytes_allocd; extern ptr_t GC_objfreelist[MAXOBJGRANULES+1]; #define beginGC_objfreelist ((ptr_t)(&GC_objfreelist)) #define endGC_objfreelist (beginGC_objfreelist+sizeof(GC_objfreelist)) extern ptr_t GC_aobjfreelist[MAXOBJGRANULES+1]; #define beginGC_aobjfreelist ((ptr_t)(&GC_aobjfreelist)) #define endGC_aobjfreelist (beginGC_aobjfreelist+sizeof(GC_aobjfreelist)) #endif #define PTRFREE 0 #define NORMAL 1 #define UNCOLLECTABLE 2 #ifdef GC_ATOMIC_UNCOLLECTABLE #define AUNCOLLECTABLE 3 #define IS_UNCOLLECTABLE(k)(((k)&~1)==UNCOLLECTABLE) #define GC_N_KINDS_INITIAL_VALUE 4 #else #define IS_UNCOLLECTABLE(k)((k)==UNCOLLECTABLE) #define GC_N_KINDS_INITIAL_VALUE 3 #endif GC_EXTERN unsigned GC_n_kinds; GC_EXTERN size_t GC_page_size; #define ROUNDUP_PAGESIZE(lb)(SIZET_SAT_ADD(lb,GC_page_size - 1)&~(GC_page_size - 1)) #ifdef MMAP_SUPPORTED #define ROUNDUP_PAGESIZE_IF_MMAP(lb)ROUNDUP_PAGESIZE(lb) #else #define ROUNDUP_PAGESIZE_IF_MMAP(lb)(lb) #endif #if defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32) GC_EXTERN SYSTEM_INFO GC_sysinfo; GC_INNER GC_bool GC_is_heap_base(void*p); #endif GC_EXTERN word GC_black_list_spacing; #ifdef GC_GCJ_SUPPORT extern struct hblk*GC_hblkfreelist[]; extern word GC_free_bytes[]; #endif GC_EXTERN word GC_root_size; GC_EXTERN GC_bool GC_debugging_started; struct blocking_data { GC_fn_type fn; void*client_data; }; struct GC_traced_stack_sect_s { ptr_t saved_stack_ptr; #ifdef IA64 ptr_t saved_backing_store_ptr; ptr_t backing_store_end; #endif struct GC_traced_stack_sect_s*prev; }; #ifdef THREADS GC_INNER void GC_push_all_stack_sections(ptr_t lo,ptr_t hi, struct GC_traced_stack_sect_s*traced_stack_sect); GC_EXTERN word GC_total_stacksize; #else GC_EXTERN ptr_t GC_blocked_sp; GC_EXTERN struct GC_traced_stack_sect_s*GC_traced_stack_sect; #endif #ifdef IA64 GC_INNER void GC_push_all_register_sections(ptr_t bs_lo,ptr_t bs_hi, int eager,struct GC_traced_stack_sect_s*traced_stack_sect); #endif #ifdef USE_MARK_BYTES #define mark_bit_from_hdr(hhdr,n)((hhdr)->hb_marks[n]) #define set_mark_bit_from_hdr(hhdr,n)((hhdr)->hb_marks[n]=1) #define clear_mark_bit_from_hdr(hhdr,n)((hhdr)->hb_marks[n]=0) #else #if defined(PARALLEL_MARK)||(defined(THREAD_SANITIZER)&&defined(THREADS)) #define OR_WORD(addr,bits)AO_or((volatile AO_t*)(addr),(AO_t)(bits)) #else #define OR_WORD(addr,bits)(void)(*(addr)|=(bits)) #endif #define mark_bit_from_hdr(hhdr,n)(((hhdr)->hb_marks[divWORDSZ(n)]>>modWORDSZ(n))&(word)1) #define set_mark_bit_from_hdr(hhdr,n)OR_WORD((hhdr)->hb_marks+divWORDSZ(n),(word)1<hb_marks[divWORDSZ(n)]&=~((word)1< MAXOBJBYTES?1:HBLK_OBJS(sz)) #else #define MARK_BIT_NO(offset,sz)BYTES_TO_GRANULES((word)(offset)) #define MARK_BIT_OFFSET(sz)BYTES_TO_GRANULES(sz) #define IF_PER_OBJ(x) #define FINAL_MARK_BIT(sz)((sz)> MAXOBJBYTES?MARK_BITS_PER_HBLK:BYTES_TO_GRANULES((sz)*HBLK_OBJS(sz))) #endif GC_INNER ptr_t GC_approx_sp(void); GC_INNER GC_bool GC_should_collect(void); void GC_apply_to_all_blocks(void (*fn)(struct hblk*h,word client_data), word client_data); GC_INNER struct hblk*GC_next_block(struct hblk*h,GC_bool allow_free); GC_INNER struct hblk*GC_prev_block(struct hblk*h); GC_INNER void GC_mark_init(void); GC_INNER void GC_clear_marks(void); GC_INNER void GC_invalidate_mark_state(void); GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame); GC_INNER void GC_initiate_gc(void); GC_INNER GC_bool GC_collection_in_progress(void); #define GC_PUSH_ALL_SYM(sym)GC_push_all(( void*)&(sym),( void*)(&(sym)+1)) GC_INNER void GC_push_all_stack(ptr_t b,ptr_t t); #if defined(WRAP_MARK_SOME)&&defined(PARALLEL_MARK) GC_INNER void GC_push_conditional_eager(void*bottom,void*top, GC_bool all); #endif GC_INNER void GC_push_roots(GC_bool all,ptr_t cold_gc_frame); GC_API_PRIV GC_push_other_roots_proc GC_push_other_roots; #ifdef THREADS void GC_push_thread_structures(void); #endif GC_EXTERN void (*GC_push_typed_structures)(void); GC_INNER void GC_with_callee_saves_pushed(void (*fn)(ptr_t,void*), volatile ptr_t arg); #if defined(SPARC)||defined(IA64) ptr_t GC_save_regs_in_stack(void); #endif #if defined(AMIGA)||defined(MACOS)||defined(GC_DARWIN_THREADS) void GC_push_one(word p); #endif #ifdef GC_WIN32_THREADS GC_INNER void GC_push_many_regs(const word*regs,unsigned count); #endif #if defined(PRINT_BLACK_LIST)||defined(KEEP_BACK_PTRS) GC_INNER void GC_mark_and_push_stack(ptr_t p,ptr_t source); #else GC_INNER void GC_mark_and_push_stack(ptr_t p); #endif GC_INNER void GC_clear_hdr_marks(hdr*hhdr); GC_INNER void GC_set_hdr_marks(hdr*hhdr); GC_INNER void GC_set_fl_marks(ptr_t p); #if defined(GC_ASSERTIONS)&&defined(THREAD_LOCAL_ALLOC) void GC_check_fl_marks(void**); #endif void GC_add_roots_inner(ptr_t b,ptr_t e,GC_bool tmp); #ifdef USE_PROC_FOR_LIBRARIES GC_INNER void GC_remove_roots_subregion(ptr_t b,ptr_t e); #endif GC_INNER void GC_exclude_static_roots_inner(void*start,void*finish); #if defined(DYNAMIC_LOADING)||defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32)||defined(PCR) GC_INNER void GC_register_dynamic_libraries(void); #endif GC_INNER void GC_cond_register_dynamic_libraries(void); ptr_t GC_get_main_stack_base(void); #ifdef IA64 GC_INNER ptr_t GC_get_register_stack_base(void); #endif void GC_register_data_segments(void); #ifdef THREADS GC_INNER void GC_thr_init(void); GC_INNER void GC_init_parallel(void); #else GC_INNER GC_bool GC_is_static_root(void*p); #ifdef TRACE_BUF void GC_add_trace_entry(char*kind,word arg1,word arg2); #endif #endif #ifdef PRINT_BLACK_LIST GC_INNER void GC_add_to_black_list_normal(word p,ptr_t source); #define GC_ADD_TO_BLACK_LIST_NORMAL(bits,source)if (GC_all_interior_pointers){ GC_add_to_black_list_stack((word)(bits),(source));} else GC_add_to_black_list_normal((word)(bits),(source)) GC_INNER void GC_add_to_black_list_stack(word p,ptr_t source); #define GC_ADD_TO_BLACK_LIST_STACK(bits,source)GC_add_to_black_list_stack((word)(bits),(source)) #else GC_INNER void GC_add_to_black_list_normal(word p); #define GC_ADD_TO_BLACK_LIST_NORMAL(bits,source)if (GC_all_interior_pointers){ GC_add_to_black_list_stack((word)(bits));} else GC_add_to_black_list_normal((word)(bits)) GC_INNER void GC_add_to_black_list_stack(word p); #define GC_ADD_TO_BLACK_LIST_STACK(bits,source)GC_add_to_black_list_stack((word)(bits)) #endif struct hblk*GC_is_black_listed(struct hblk*h,word len); GC_INNER void GC_promote_black_lists(void); GC_INNER void GC_unpromote_black_lists(void); GC_INNER ptr_t GC_scratch_alloc(size_t bytes); #ifdef GWW_VDB #else #define GC_scratch_recycle_no_gww GC_scratch_recycle_inner #endif GC_INNER void GC_scratch_recycle_inner(void*ptr,size_t bytes); #ifdef MARK_BIT_PER_GRANULE GC_INNER GC_bool GC_add_map_entry(size_t sz); #endif GC_INNER void GC_register_displacement_inner(size_t offset); GC_INNER void GC_new_hblk(size_t size_in_granules,int kind); GC_INNER ptr_t GC_build_fl(struct hblk*h,size_t words,GC_bool clear, ptr_t list); GC_INNER struct hblk*GC_allochblk(size_t size_in_bytes,int kind, unsigned flags); GC_INNER ptr_t GC_alloc_large(size_t lb,int k,unsigned flags); GC_INNER void GC_freehblk(struct hblk*p); GC_INNER GC_bool GC_expand_hp_inner(word n); GC_INNER void GC_start_reclaim(GC_bool abort_if_found); GC_INNER void GC_continue_reclaim(word sz,int kind); GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func,GC_bool ignore_old); GC_INNER ptr_t GC_reclaim_generic(struct hblk*hbp,hdr*hhdr,size_t sz, GC_bool init,ptr_t list, signed_word*count); GC_INNER GC_bool GC_block_empty(hdr*hhdr); GC_INNER int GC_CALLBACK GC_never_stop_func(void); GC_INNER GC_bool GC_try_to_collect_inner(GC_stop_func f); #define GC_gcollect_inner()(void)GC_try_to_collect_inner(GC_never_stop_func) #ifdef THREADS GC_EXTERN GC_bool GC_in_thread_creation; #endif GC_EXTERN GC_bool GC_is_initialized; GC_INNER void GC_collect_a_little_inner(int n); GC_INNER void*GC_generic_malloc_inner(size_t lb,int k); #if defined(DBG_HDRS_ALL)||defined(GC_GCJ_SUPPORT)||!defined(GC_NO_FINALIZATION) GC_INNER void*GC_generic_malloc_inner_ignore_off_page(size_t lb,int k); #endif GC_INNER GC_bool GC_collect_or_expand(word needed_blocks, GC_bool ignore_off_page,GC_bool retry); GC_INNER ptr_t GC_allocobj(size_t sz,int kind); #ifdef GC_ADD_CALLER #ifdef GC_HAVE_RETURN_ADDR_PARENT #define GC_DBG_EXTRAS GC_RETURN_ADDR_PARENT,NULL,0 #else #define GC_DBG_EXTRAS GC_RETURN_ADDR,NULL,0 #endif #else #define GC_DBG_EXTRAS "unknown",0 #endif #ifdef GC_COLLECT_AT_MALLOC extern size_t GC_dbg_collect_at_malloc_min_lb; #define GC_DBG_COLLECT_AT_MALLOC(lb)(void)((lb)>=GC_dbg_collect_at_malloc_min_lb?(GC_gcollect(),0):0) #else #define GC_DBG_COLLECT_AT_MALLOC(lb)(void)0 #endif #if defined(THREAD_LOCAL_ALLOC)&&defined(GC_GCJ_SUPPORT) GC_INNER void*GC_core_gcj_malloc(size_t,void*); #endif GC_INNER void GC_init_headers(void); GC_INNER struct hblkhdr*GC_install_header(struct hblk*h); GC_INNER GC_bool GC_install_counts(struct hblk*h,size_t sz); GC_INNER void GC_remove_header(struct hblk*h); GC_INNER void GC_remove_counts(struct hblk*h,size_t sz); GC_INNER hdr*GC_find_header(ptr_t h); GC_INNER void GC_add_to_heap(struct hblk*p,size_t bytes); #ifdef USE_PROC_FOR_LIBRARIES GC_INNER void GC_add_to_our_memory(ptr_t p,size_t bytes); #else #define GC_add_to_our_memory(p,bytes) #endif GC_INNER void GC_print_all_errors(void); GC_EXTERN void (*GC_check_heap)(void); GC_EXTERN void (*GC_print_all_smashed)(void); GC_EXTERN void (*GC_print_heap_obj)(ptr_t p); #if defined(LINUX)&&defined(__ELF__)&&!defined(SMALL_CONFIG) void GC_print_address_map(void); #endif #ifndef SHORT_DBG_HDRS GC_EXTERN GC_bool GC_findleak_delay_free; GC_INNER GC_bool GC_check_leaked(ptr_t base); #endif GC_EXTERN GC_bool GC_have_errors; #define VERBOSE 2 #if!defined(NO_CLOCK)||!defined(SMALL_CONFIG) extern int GC_print_stats; #else #define GC_print_stats 0 #endif #ifdef KEEP_BACK_PTRS GC_EXTERN long GC_backtraces; GC_INNER void GC_generate_random_backtrace_no_gc(void); #endif #ifdef LINT2 #define GC_RAND_MAX (~0U>>1) GC_API_PRIV long GC_random(void); #endif GC_EXTERN GC_bool GC_print_back_height; #ifdef MAKE_BACK_GRAPH void GC_print_back_graph_stats(void); #endif #ifdef THREADS GC_INNER void GC_free_inner(void*p); #endif #ifdef DBG_HDRS_ALL GC_INNER void*GC_debug_generic_malloc_inner(size_t lb,int k); GC_INNER void*GC_debug_generic_malloc_inner_ignore_off_page(size_t lb, int k); #define GC_INTERNAL_MALLOC GC_debug_generic_malloc_inner #define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE GC_debug_generic_malloc_inner_ignore_off_page #ifdef THREADS GC_INNER void GC_debug_free_inner(void*p); #define GC_INTERNAL_FREE GC_debug_free_inner #else #define GC_INTERNAL_FREE GC_debug_free #endif #else #define GC_INTERNAL_MALLOC GC_generic_malloc_inner #define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE GC_generic_malloc_inner_ignore_off_page #ifdef THREADS #define GC_INTERNAL_FREE GC_free_inner #else #define GC_INTERNAL_FREE GC_free #endif #endif #ifdef USE_MUNMAP GC_INNER void GC_unmap_old(void); GC_INNER void GC_merge_unmapped(void); GC_INNER void GC_unmap(ptr_t start,size_t bytes); GC_INNER void GC_remap(ptr_t start,size_t bytes); GC_INNER void GC_unmap_gap(ptr_t start1,size_t bytes1,ptr_t start2, size_t bytes2); GC_INLINE ptr_t GC_unmap_end(ptr_t start,size_t bytes) { return (ptr_t)((word)(start+bytes)&~(GC_page_size - 1)); } #endif #ifdef CAN_HANDLE_FORK GC_EXTERN int GC_handle_fork; #endif #ifdef GC_DISABLE_INCREMENTAL #define GC_incremental FALSE #define GC_auto_incremental FALSE #define GC_manual_vdb FALSE #define GC_dirty(p)(void)(p) #define REACHABLE_AFTER_DIRTY(p)(void)(p) #else GC_EXTERN GC_bool GC_incremental; GC_INNER void GC_read_dirty(GC_bool output_unneeded); GC_INNER GC_bool GC_page_was_dirty(struct hblk*h); GC_INNER void GC_remove_protection(struct hblk*h,word nblocks, GC_bool pointerfree); GC_INNER GC_bool GC_dirty_init(void); GC_EXTERN GC_bool GC_manual_vdb; #define GC_auto_incremental (GC_incremental&&!GC_manual_vdb) GC_INNER void GC_dirty_inner(const void*p); #define GC_dirty(p)(GC_manual_vdb?GC_dirty_inner(p):(void)0) #define REACHABLE_AFTER_DIRTY(p)GC_reachable_here(p) #endif #define GC_base_C(p)((const void*)GC_base(( void*)(p))) void GC_print_block_list(void); void GC_print_hblkfreelist(void); void GC_print_heap_sects(void); void GC_print_static_roots(void); #ifdef KEEP_BACK_PTRS GC_INNER void GC_store_back_pointer(ptr_t source,ptr_t dest); GC_INNER void GC_marked_for_finalization(ptr_t dest); #define GC_STORE_BACK_PTR(source,dest)GC_store_back_pointer(source,dest) #define GC_MARKED_FOR_FINALIZATION(dest)GC_marked_for_finalization(dest) #else #define GC_STORE_BACK_PTR(source,dest)(void)(source) #define GC_MARKED_FOR_FINALIZATION(dest) #endif void GC_noop6(word,word,word,word,word,word); GC_API void GC_CALL GC_noop1(word); #ifndef GC_ATTR_FORMAT_PRINTF #if GC_GNUC_PREREQ(3,0) #define GC_ATTR_FORMAT_PRINTF(spec_argnum,first_checked)__attribute__((__format__(__printf__,spec_argnum,first_checked))) #else #define GC_ATTR_FORMAT_PRINTF(spec_argnum,first_checked) #endif #endif GC_API_PRIV void GC_printf(const char*format,...) GC_ATTR_FORMAT_PRINTF(1,2); GC_API_PRIV void GC_err_printf(const char*format,...) GC_ATTR_FORMAT_PRINTF(1,2); GC_API_PRIV void GC_log_printf(const char*format,...) GC_ATTR_FORMAT_PRINTF(1,2); #ifndef GC_ANDROID_LOG #define GC_PRINT_STATS_FLAG (GC_print_stats!=0) #define GC_INFOLOG_PRINTF GC_COND_LOG_PRINTF #define GC_verbose_log_printf GC_log_printf #else extern GC_bool GC_quiet; #define GC_PRINT_STATS_FLAG (!GC_quiet) #ifndef GC_INFOLOG_PRINTF #define GC_INFOLOG_PRINTF if (GC_quiet){} else GC_info_log_printf #endif GC_INNER void GC_info_log_printf(const char*format,...) GC_ATTR_FORMAT_PRINTF(1,2); GC_INNER void GC_verbose_log_printf(const char*format,...) GC_ATTR_FORMAT_PRINTF(1,2); #endif #define GC_COND_LOG_PRINTF if (EXPECT(!GC_print_stats,TRUE)){} else GC_log_printf #define GC_VERBOSE_LOG_PRINTF if (EXPECT(GC_print_stats!=VERBOSE,TRUE)){} else GC_verbose_log_printf #ifndef GC_DBGLOG_PRINTF #define GC_DBGLOG_PRINTF if (!GC_PRINT_STATS_FLAG){} else GC_log_printf #endif void GC_err_puts(const char*s); #define TO_KiB_UL(v)((unsigned long)(((v)+((1<<9)- 1))>>10)) GC_EXTERN unsigned GC_fail_count; GC_EXTERN long GC_large_alloc_warn_interval; GC_EXTERN signed_word GC_bytes_found; #ifndef GC_GET_HEAP_USAGE_NOT_NEEDED GC_EXTERN word GC_reclaimed_bytes_before_gc; #endif #ifdef USE_MUNMAP GC_EXTERN int GC_unmap_threshold; GC_EXTERN GC_bool GC_force_unmap_on_gcollect; #endif #ifdef MSWIN32 GC_EXTERN GC_bool GC_no_win32_dlls; GC_EXTERN GC_bool GC_wnt; #endif #ifdef THREADS #if (defined(MSWIN32)&&!defined(CONSOLE_LOG))||defined(MSWINCE) GC_EXTERN CRITICAL_SECTION GC_write_cs; #ifdef GC_ASSERTIONS GC_EXTERN GC_bool GC_write_disabled; #endif #endif #if defined(GC_DISABLE_INCREMENTAL)||defined(HAVE_LOCKFREE_AO_OR) #define GC_acquire_dirty_lock()(void)0 #define GC_release_dirty_lock()(void)0 #else #define GC_acquire_dirty_lock()do { } while (AO_test_and_set_acquire(&GC_fault_handler_lock)==AO_TS_SET) #define GC_release_dirty_lock()AO_CLEAR(&GC_fault_handler_lock) GC_EXTERN volatile AO_TS_t GC_fault_handler_lock; #endif #ifdef MSWINCE GC_EXTERN GC_bool GC_dont_query_stack_min; #endif #elif defined(IA64) GC_EXTERN ptr_t GC_save_regs_ret_val; #endif #ifdef THREAD_LOCAL_ALLOC GC_EXTERN GC_bool GC_world_stopped; GC_INNER void GC_mark_thread_local_free_lists(void); #endif #if defined(MPROTECT_VDB)&&defined(GWW_VDB) GC_INNER GC_bool GC_gww_dirty_init(void); #endif #if defined(CHECKSUMS)||defined(PROC_VDB) GC_INNER GC_bool GC_page_was_ever_dirty(struct hblk*h); #endif #ifdef CHECKSUMS #if defined(MPROTECT_VDB)&&!defined(DARWIN) void GC_record_fault(struct hblk*h); #endif void GC_check_dirty(void); #endif GC_INNER void GC_default_print_heap_obj_proc(ptr_t p); GC_INNER void GC_setpagesize(void); GC_INNER void GC_initialize_offsets(void); GC_INNER void GC_bl_init(void); GC_INNER void GC_bl_init_no_interiors(void); GC_INNER void GC_start_debugging_inner(void); GC_INNER void*GC_store_debug_info_inner(void*p,word sz,const char*str, int linenum); #ifdef REDIRECT_MALLOC #ifdef GC_LINUX_THREADS GC_INNER GC_bool GC_text_mapping(char*nm,ptr_t*startp,ptr_t*endp); #endif #elif defined(USE_WINALLOC) GC_INNER void GC_add_current_malloc_heap(void); #endif #ifdef MAKE_BACK_GRAPH GC_INNER void GC_build_back_graph(void); GC_INNER void GC_traverse_back_graph(void); #endif #ifdef MSWIN32 GC_INNER void GC_init_win32(void); #endif #if!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(CYGWIN32) GC_INNER void*GC_roots_present(ptr_t); #endif #ifdef GC_WIN32_THREADS GC_INNER void GC_get_next_stack(char*start,char*limit,char**lo, char**hi); #if defined(MPROTECT_VDB)&&!defined(CYGWIN32) GC_INNER void GC_set_write_fault_handler(void); #endif #if defined(WRAP_MARK_SOME)&&!defined(GC_PTHREADS) GC_INNER GC_bool GC_started_thread_while_stopped(void); #endif #endif #ifdef THREADS GC_INNER void GC_reset_finalizer_nested(void); GC_INNER unsigned char*GC_check_finalizer_nested(void); GC_INNER void GC_do_blocking_inner(ptr_t data,void*context); GC_INNER void GC_push_all_stacks(void); #ifdef USE_PROC_FOR_LIBRARIES GC_INNER GC_bool GC_segment_is_thread_stack(ptr_t lo,ptr_t hi); #endif #ifdef IA64 GC_INNER ptr_t GC_greatest_stack_base_below(ptr_t bound); #endif #endif #ifdef DYNAMIC_LOADING GC_INNER GC_bool GC_register_main_static_data(void); #ifdef DARWIN GC_INNER void GC_init_dyld(void); #endif #endif #ifdef SEARCH_FOR_DATA_START GC_INNER void GC_init_linux_data_start(void); void*GC_find_limit(void*,int); #endif #if defined(NETBSD)&&defined(__ELF__) GC_INNER void GC_init_netbsd_elf(void); void*GC_find_limit(void*,int); #endif #ifdef UNIX_LIKE GC_INNER void GC_set_and_save_fault_handler(void (*handler)(int)); #endif #ifdef NEED_PROC_MAPS #if defined(DYNAMIC_LOADING)&&defined(USE_PROC_FOR_LIBRARIES) GC_INNER char*GC_parse_map_entry(char*buf_ptr,ptr_t*start,ptr_t*end, char**prot,unsigned int*maj_dev, char**mapping_name); #endif #if defined(IA64)||defined(INCLUDE_LINUX_THREAD_DESCR) GC_INNER GC_bool GC_enclosing_mapping(ptr_t addr, ptr_t*startp,ptr_t*endp); #endif GC_INNER char*GC_get_maps(void); #endif #ifdef GC_ASSERTIONS #define GC_ASSERT(expr)do { if (!(expr)){ GC_err_printf("Assertion failure:%s:%d\n",__FILE__,__LINE__);ABORT("assertion failure");} } while (0) GC_INNER word GC_compute_large_free_bytes(void); GC_INNER word GC_compute_root_size(void); #else #define GC_ASSERT(expr) #endif #if _MSC_VER>=1700 #define GC_STATIC_ASSERT(expr)static_assert(expr,"static assertion failed:" #expr) #elif defined(static_assert)&&__STDC_VERSION__>=201112L #define GC_STATIC_ASSERT(expr)static_assert(expr,#expr) #elif defined(mips)&&!defined(__GNUC__) #define GC_STATIC_ASSERT(expr)do { if (0){ char j[(expr)?1:-1];j[0]='\0';j[0]=j[0];} } while(0) #else #define GC_STATIC_ASSERT(expr)(void)sizeof(char[(expr)?1:-1]) #endif #if GC_GNUC_PREREQ(4,0) #define NONNULL_ARG_NOT_NULL(arg)(*(volatile void**)&(arg)!=NULL) #else #define NONNULL_ARG_NOT_NULL(arg)(NULL!=(arg)) #endif #define COND_DUMP_CHECKS do { GC_ASSERT(GC_compute_large_free_bytes()==GC_large_free_bytes);GC_ASSERT(GC_compute_root_size()==GC_root_size);} while (0) #ifndef NO_DEBUGGING GC_EXTERN GC_bool GC_dump_regularly; #define COND_DUMP if (EXPECT(GC_dump_regularly,FALSE)){ GC_dump_named(NULL);} else COND_DUMP_CHECKS #else #define COND_DUMP COND_DUMP_CHECKS #endif #if defined(PARALLEL_MARK) #define GC_markers_m1 GC_parallel GC_EXTERN GC_bool GC_parallel_mark_disabled; GC_INNER void GC_wait_for_markers_init(void); GC_INNER void GC_acquire_mark_lock(void); GC_INNER void GC_release_mark_lock(void); GC_INNER void GC_notify_all_builder(void); GC_INNER void GC_wait_for_reclaim(void); GC_EXTERN signed_word GC_fl_builder_count; GC_INNER void GC_notify_all_marker(void); GC_INNER void GC_wait_marker(void); GC_EXTERN word GC_mark_no; GC_INNER void GC_help_marker(word my_mark_no); GC_INNER void GC_start_mark_threads_inner(void); #endif #if defined(GC_PTHREADS)&&!defined(GC_WIN32_THREADS)&&!defined(NACL)&&!defined(GC_DARWIN_THREADS)&&!defined(SIG_SUSPEND) #if (defined(GC_LINUX_THREADS)||defined(GC_DGUX386_THREADS))&&!defined(GC_USESIGRT_SIGNALS) #if defined(SPARC)&&!defined(SIGPWR) #define SIG_SUSPEND SIGLOST #else #define SIG_SUSPEND SIGPWR #endif #elif defined(GC_OPENBSD_THREADS) #ifndef GC_OPENBSD_UTHREADS #define SIG_SUSPEND SIGXFSZ #endif #elif defined(_SIGRTMIN)&&!defined(CPPCHECK) #define SIG_SUSPEND _SIGRTMIN+6 #else #define SIG_SUSPEND SIGRTMIN+6 #endif #endif #if defined(GC_PTHREADS)&&!defined(GC_SEM_INIT_PSHARED) #define GC_SEM_INIT_PSHARED 0 #endif #if (defined(UNIX_LIKE)||(defined(NEED_FIND_LIMIT)&&defined(CYGWIN32)))&&!defined(GC_NO_SIGSETJMP) #if defined(SUNOS5SIGS)&&!defined(FREEBSD)&&!defined(LINUX) EXTERN_C_END #include EXTERN_C_BEGIN #endif #define SETJMP(env)sigsetjmp(env,1) #define LONGJMP(env,val)siglongjmp(env,val) #define JMP_BUF sigjmp_buf #else #ifdef ECOS #define SETJMP(env)hal_setjmp(env) #else #define SETJMP(env)setjmp(env) #endif #define LONGJMP(env,val)longjmp(env,val) #define JMP_BUF jmp_buf #endif #if defined(HEURISTIC2)||defined(SEARCH_FOR_DATA_START)||((defined(SVR4)||defined(AIX)||defined(DGUX)||(defined(LINUX)&&defined(SPARC)))&&!defined(PCR)) #define NEED_FIND_LIMIT #endif #if defined(DATASTART_USES_BSDGETDATASTART) EXTERN_C_END #include EXTERN_C_BEGIN #if!defined(PCR) #define NEED_FIND_LIMIT #endif GC_INNER ptr_t GC_FreeBSDGetDataStart(size_t,ptr_t); #define DATASTART_IS_FUNC #endif #if (defined(NETBSD)||defined(OPENBSD))&&defined(__ELF__)&&!defined(NEED_FIND_LIMIT) #define NEED_FIND_LIMIT #endif #if defined(IA64)&&!defined(NEED_FIND_LIMIT) #define NEED_FIND_LIMIT #endif #if defined(NEED_FIND_LIMIT)||(defined(USE_PROC_FOR_LIBRARIES)&&defined(THREADS)) GC_EXTERN JMP_BUF GC_jmp_buf; GC_INNER void GC_setup_temporary_fault_handler(void); GC_INNER void GC_reset_fault_handler(void); #endif #if defined(CANCEL_SAFE) #if defined(GC_ASSERTIONS)&&(defined(USE_COMPILER_TLS)||(defined(LINUX)&&!defined(ARM32)&&GC_GNUC_PREREQ(3,3)||defined(HPUX))) extern __thread unsigned char GC_cancel_disable_count; #define NEED_CANCEL_DISABLE_COUNT #define INCR_CANCEL_DISABLE()++GC_cancel_disable_count #define DECR_CANCEL_DISABLE()--GC_cancel_disable_count #define ASSERT_CANCEL_DISABLED()GC_ASSERT(GC_cancel_disable_count > 0) #else #define INCR_CANCEL_DISABLE() #define DECR_CANCEL_DISABLE() #define ASSERT_CANCEL_DISABLED()(void)0 #endif #define DISABLE_CANCEL(state)do { pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,&state);INCR_CANCEL_DISABLE();} while (0) #define RESTORE_CANCEL(state)do { ASSERT_CANCEL_DISABLED();pthread_setcancelstate(state,NULL);DECR_CANCEL_DISABLE();} while (0) #else #define DISABLE_CANCEL(state)(void)0 #define RESTORE_CANCEL(state)(void)0 #define ASSERT_CANCEL_DISABLED()(void)0 #endif EXTERN_C_END #endif #ifdef KEEP_BACK_PTRS #ifndef GC_BACKPTR_H #define GC_BACKPTR_H #ifndef GC_H #endif #ifdef __cplusplus extern "C" { #endif typedef enum { GC_UNREFERENCED, GC_NO_SPACE, GC_REFD_FROM_ROOT, GC_REFD_FROM_REG, GC_REFD_FROM_HEAP, GC_FINALIZER_REFD } GC_ref_kind; GC_API GC_ref_kind GC_CALL GC_get_back_ptr_info(void*, void**,size_t*) GC_ATTR_NONNULL(1); GC_API void*GC_CALL GC_generate_random_heap_address(void); GC_API void*GC_CALL GC_generate_random_valid_address(void); GC_API void GC_CALL GC_generate_random_backtrace(void); GC_API void GC_CALL GC_print_backtrace(void*)GC_ATTR_NONNULL(1); #ifdef __cplusplus } #endif #endif #endif EXTERN_C_BEGIN #if CPP_WORDSZ==32 #define START_FLAG (word)0xfedcedcb #define END_FLAG (word)0xbcdecdef #else #define START_FLAG GC_WORD_C(0xFEDCEDCBfedcedcb) #define END_FLAG GC_WORD_C(0xBCDECDEFbcdecdef) #endif #if defined(KEEP_BACK_PTRS)||defined(PRINT_BLACK_LIST)||defined(MAKE_BACK_GRAPH) #define NOT_MARKED (ptr_t)0 #define MARKED_FOR_FINALIZATION ((ptr_t)(word)2) #define MARKED_FROM_REGISTER ((ptr_t)(word)4) #endif typedef struct { #if defined(KEEP_BACK_PTRS)||defined(MAKE_BACK_GRAPH) #if ALIGNMENT==1 #define HIDE_BACK_PTR(p)GC_HIDE_POINTER(~1&(word)(p)) #else #define HIDE_BACK_PTR(p)GC_HIDE_POINTER(p) #endif #ifdef KEEP_BACK_PTRS GC_hidden_pointer oh_back_ptr; #endif #ifdef MAKE_BACK_GRAPH GC_hidden_pointer oh_bg_ptr; #endif #if defined(KEEP_BACK_PTRS)!=defined(MAKE_BACK_GRAPH) word oh_dummy; #endif #endif const char*oh_string; signed_word oh_int; #ifdef NEED_CALLINFO struct callinfo oh_ci[NFRAMES]; #endif #ifndef SHORT_DBG_HDRS word oh_sz; word oh_sf; #endif } oh; #ifdef SHORT_DBG_HDRS #define DEBUG_BYTES (sizeof (oh)) #define UNCOLLECTABLE_DEBUG_BYTES DEBUG_BYTES #else #define UNCOLLECTABLE_DEBUG_BYTES (sizeof (oh)+sizeof (word)) #define DEBUG_BYTES (UNCOLLECTABLE_DEBUG_BYTES - EXTRA_BYTES) #endif #define SIMPLE_ROUNDED_UP_WORDS(n)BYTES_TO_WORDS((n)+WORDS_TO_BYTES(1)- 1) #if defined(SAVE_CALL_CHAIN) struct callinfo; GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]); GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]); #define ADD_CALL_CHAIN(base,ra)GC_save_callers(((oh*)(base))->oh_ci) #define PRINT_CALL_CHAIN(base)GC_print_callers(((oh*)(base))->oh_ci) #elif defined(GC_ADD_CALLER) struct callinfo; GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]); #define ADD_CALL_CHAIN(base,ra)((oh*)(base))->oh_ci[0].ci_pc=(ra) #define PRINT_CALL_CHAIN(base)GC_print_callers(((oh*)(base))->oh_ci) #else #define ADD_CALL_CHAIN(base,ra) #define PRINT_CALL_CHAIN(base) #endif #ifdef GC_ADD_CALLER #define OPT_RA ra, #else #define OPT_RA #endif #ifdef SHORT_DBG_HDRS #define GC_has_other_debug_info(p)1 #else GC_INNER int GC_has_other_debug_info(ptr_t p); #endif #if defined(KEEP_BACK_PTRS)||defined(MAKE_BACK_GRAPH) #if defined(SHORT_DBG_HDRS)&&!defined(CPPCHECK) #error Non-ptr stored in object results in GC_HAS_DEBUG_INFO malfunction #endif #if defined(PARALLEL_MARK)&&defined(KEEP_BACK_PTRS) #define GC_HAS_DEBUG_INFO(p)((AO_load((volatile AO_t*)(p))&1)!=0&&GC_has_other_debug_info(p)> 0) #else #define GC_HAS_DEBUG_INFO(p)((*(word*)(p)&1)&&GC_has_other_debug_info(p)> 0) #endif #else #define GC_HAS_DEBUG_INFO(p)(GC_has_other_debug_info(p)> 0) #endif EXTERN_C_END #endif #ifdef MAKE_BACK_GRAPH #define MAX_IN 10 #if (!defined(DBG_HDRS_ALL)||(ALIGNMENT!=CPP_WORDSZ/8))&&!defined(CPPCHECK) #error The configuration does not support MAKE_BACK_GRAPH #endif #define FLAG_MANY 2 typedef struct back_edges_struct { word n_edges; unsigned short flags; #define RETAIN 1 unsigned short height_gc_no; signed_word height; #define HEIGHT_UNKNOWN (-2) #define HEIGHT_IN_PROGRESS (-1) ptr_t edges[MAX_IN]; struct back_edges_struct*cont; } back_edges; #define MAX_BACK_EDGE_STRUCTS 100000 static back_edges*back_edge_space=0; STATIC int GC_n_back_edge_structs=0; static back_edges*avail_back_edges=0; static back_edges*new_back_edges(void) { if (0==back_edge_space){ size_t bytes_to_get=ROUNDUP_PAGESIZE_IF_MMAP(MAX_BACK_EDGE_STRUCTS *sizeof(back_edges)); GC_ASSERT(GC_page_size!=0); back_edge_space=(back_edges*)GET_MEM(bytes_to_get); if (NULL==back_edge_space) ABORT("Insufficient memory for back edges"); GC_add_to_our_memory((ptr_t)back_edge_space,bytes_to_get); } if (0!=avail_back_edges){ back_edges*result=avail_back_edges; avail_back_edges=result->cont; result->cont=0; return result; } if (GC_n_back_edge_structs>=MAX_BACK_EDGE_STRUCTS - 1){ ABORT("Needed too much space for back edges:adjust " "MAX_BACK_EDGE_STRUCTS"); } return back_edge_space+(GC_n_back_edge_structs++); } static void deallocate_back_edges(back_edges*p) { back_edges*last=p; while (0!=last->cont)last=last->cont; last->cont=avail_back_edges; avail_back_edges=p; } #define INITIAL_IN_PROGRESS 10000 static ptr_t*in_progress_space=0; static size_t in_progress_size=0; static size_t n_in_progress=0; static void push_in_progress(ptr_t p) { if (n_in_progress>=in_progress_size){ ptr_t*new_in_progress_space; GC_ASSERT(GC_page_size!=0); if (NULL==in_progress_space){ in_progress_size=ROUNDUP_PAGESIZE_IF_MMAP(INITIAL_IN_PROGRESS *sizeof(ptr_t)) /sizeof(ptr_t); new_in_progress_space= (ptr_t*)GET_MEM(in_progress_size*sizeof(ptr_t)); } else { in_progress_size*=2; new_in_progress_space=(ptr_t*) GET_MEM(in_progress_size*sizeof(ptr_t)); if (new_in_progress_space!=NULL) BCOPY(in_progress_space,new_in_progress_space, n_in_progress*sizeof(ptr_t)); } GC_add_to_our_memory((ptr_t)new_in_progress_space, in_progress_size*sizeof(ptr_t)); #ifndef GWW_VDB GC_scratch_recycle_no_gww(in_progress_space, n_in_progress*sizeof(ptr_t)); #elif defined(LINT2) GC_noop1((word)in_progress_space); #endif in_progress_space=new_in_progress_space; } if (in_progress_space==0) ABORT("MAKE_BACK_GRAPH:Out of in-progress space:" "Huge linear data structure?"); in_progress_space[n_in_progress++]=p; } static GC_bool is_in_progress(ptr_t p) { size_t i; for (i=0;i < n_in_progress;++i){ if (in_progress_space[i]==p)return TRUE; } return FALSE; } GC_INLINE void pop_in_progress(ptr_t p GC_ATTR_UNUSED) { --n_in_progress; GC_ASSERT(in_progress_space[n_in_progress]==p); } #define GET_OH_BG_PTR(p)(ptr_t)GC_REVEAL_POINTER(((oh*)(p))->oh_bg_ptr) #define SET_OH_BG_PTR(p,q)(((oh*)(p))->oh_bg_ptr=GC_HIDE_POINTER(q)) static void ensure_struct(ptr_t p) { ptr_t old_back_ptr=GET_OH_BG_PTR(p); if (!((word)old_back_ptr&FLAG_MANY)){ back_edges*be=new_back_edges(); be->flags=0; if (0==old_back_ptr){ be->n_edges=0; } else { be->n_edges=1; be->edges[0]=old_back_ptr; } be->height=HEIGHT_UNKNOWN; be->height_gc_no=(unsigned short)(GC_gc_no - 1); GC_ASSERT((word)be>=(word)back_edge_space); SET_OH_BG_PTR(p,(word)be|FLAG_MANY); } } static void add_edge(ptr_t p,ptr_t q) { ptr_t pred=GET_OH_BG_PTR(q); back_edges*be,*be_cont; word i; GC_ASSERT(p==GC_base(p)&&q==GC_base(q)); if (!GC_HAS_DEBUG_INFO(q)||!GC_HAS_DEBUG_INFO(p)){ return; } if (NULL==pred){ static unsigned random_number=13; #define GOT_LUCKY_NUMBER (((++random_number)&0x7f)==0) SET_OH_BG_PTR(q,p); if (GOT_LUCKY_NUMBER)ensure_struct(q); return; } { back_edges*e=(back_edges*)((word)pred&~FLAG_MANY); word n_edges; word total; int local=0; if (((word)pred&FLAG_MANY)!=0){ n_edges=e->n_edges; } else if (((word)COVERT_DATAFLOW(pred)&1)==0){ n_edges=1; local=-1; } else { n_edges=0; } for (total=0;total < n_edges;++total){ if (local==MAX_IN){ e=e->cont; local=0; } if (local>=0) pred=e->edges[local++]; if (pred==p) return; } } ensure_struct(q); be=(back_edges*)((word)GET_OH_BG_PTR(q)&~FLAG_MANY); for (i=be->n_edges,be_cont=be;i > MAX_IN;i-=MAX_IN) be_cont=be_cont->cont; if (i==MAX_IN){ be_cont->cont=new_back_edges(); be_cont=be_cont->cont; i=0; } be_cont->edges[i]=p; be->n_edges++; #ifdef DEBUG_PRINT_BIG_N_EDGES if (GC_print_stats==VERBOSE&&be->n_edges==100){ GC_err_printf("The following object has big in-degree:\n"); GC_print_heap_obj(q); } #endif } typedef void (*per_object_func)(ptr_t p,size_t n_bytes,word gc_descr); static void per_object_helper(struct hblk*h,word fn) { hdr*hhdr=HDR(h); size_t sz=(size_t)hhdr->hb_sz; word descr=hhdr->hb_descr; per_object_func f=(per_object_func)fn; size_t i=0; do { f((ptr_t)(h->hb_body+i),sz,descr); i+=sz; } while (i+sz<=BYTES_TO_WORDS(HBLKSIZE)); } GC_INLINE void GC_apply_to_each_object(per_object_func f) { GC_apply_to_all_blocks(per_object_helper,(word)f); } static void reset_back_edge(ptr_t p,size_t n_bytes GC_ATTR_UNUSED, word gc_descr GC_ATTR_UNUSED) { if (GC_HAS_DEBUG_INFO(p)){ ptr_t old_back_ptr=GET_OH_BG_PTR(p); if ((word)old_back_ptr&FLAG_MANY){ back_edges*be=(back_edges*)((word)old_back_ptr&~FLAG_MANY); if (!(be->flags&RETAIN)){ deallocate_back_edges(be); SET_OH_BG_PTR(p,0); } else { GC_ASSERT(GC_is_marked(p)); be->n_edges=0; if (0!=be->cont){ deallocate_back_edges(be->cont); be->cont=0; } GC_ASSERT(GC_is_marked(p)); be->flags&=~RETAIN; } } else { SET_OH_BG_PTR(p,0); } } } static void add_back_edges(ptr_t p,size_t n_bytes,word gc_descr) { word*currentp=(word*)(p+sizeof(oh)); if((gc_descr&GC_DS_TAGS)!=GC_DS_LENGTH){ gc_descr=n_bytes; } while ((word)currentp < (word)(p+gc_descr)){ word current=*currentp++; FIXUP_POINTER(current); if (current>=(word)GC_least_plausible_heap_addr&& current<=(word)GC_greatest_plausible_heap_addr){ ptr_t target=(ptr_t)GC_base((void*)current); if (0!=target){ add_edge(p,target); } } } } GC_INNER void GC_build_back_graph(void) { GC_ASSERT(I_HOLD_LOCK()); GC_apply_to_each_object(add_back_edges); } static word backwards_height(ptr_t p) { word result; ptr_t pred=GET_OH_BG_PTR(p); back_edges*be; if (NULL==pred) return 1; if (((word)pred&FLAG_MANY)==0){ if (is_in_progress(p))return 0; push_in_progress(p); result=backwards_height(pred)+1; pop_in_progress(p); return result; } be=(back_edges*)((word)pred&~FLAG_MANY); if (be->height>=0&&be->height_gc_no==(unsigned short)GC_gc_no) return be->height; if (be->height==HEIGHT_IN_PROGRESS)return 0; result=(be->height > 0?be->height:1); be->height=HEIGHT_IN_PROGRESS; { back_edges*e=be; word n_edges; word total; int local=0; if (((word)pred&FLAG_MANY)!=0){ n_edges=e->n_edges; } else if (((word)pred&1)==0){ n_edges=1; local=-1; } else { n_edges=0; } for (total=0;total < n_edges;++total){ word this_height; if (local==MAX_IN){ e=e->cont; local=0; } if (local>=0) pred=e->edges[local++]; if (GC_is_marked(pred)&&((word)GET_OH_BG_PTR(p)&FLAG_MANY)==0){ GC_COND_LOG_PRINTF("Found bogus pointer from %p to %p\n", (void*)pred,(void*)p); this_height=1; } else { this_height=backwards_height(pred); } if (this_height>=result) result=this_height+1; } } be->height=result; be->height_gc_no=(unsigned short)GC_gc_no; return result; } STATIC word GC_max_height=0; STATIC ptr_t GC_deepest_obj=NULL; static void update_max_height(ptr_t p,size_t n_bytes GC_ATTR_UNUSED, word gc_descr GC_ATTR_UNUSED) { if (GC_is_marked(p)&&GC_HAS_DEBUG_INFO(p)){ word p_height=0; ptr_t p_deepest_obj=0; ptr_t back_ptr; back_edges*be=0; back_ptr=GET_OH_BG_PTR(p); if (0!=back_ptr&&((word)back_ptr&FLAG_MANY)){ be=(back_edges*)((word)back_ptr&~FLAG_MANY); if (be->height!=HEIGHT_UNKNOWN)p_height=be->height; } { ptr_t pred=GET_OH_BG_PTR(p); back_edges*e=(back_edges*)((word)pred&~FLAG_MANY); word n_edges; word total; int local=0; if (((word)pred&FLAG_MANY)!=0){ n_edges=e->n_edges; } else if (pred!=NULL&&((word)pred&1)==0){ n_edges=1; local=-1; } else { n_edges=0; } for (total=0;total < n_edges;++total){ if (local==MAX_IN){ e=e->cont; local=0; } if (local>=0) pred=e->edges[local++]; if (!GC_is_marked(pred)&&GC_HAS_DEBUG_INFO(pred)){ word this_height=backwards_height(pred); if (this_height > p_height){ p_height=this_height; p_deepest_obj=pred; } } } } if (p_height > 0){ if (be==0){ ensure_struct(p); back_ptr=GET_OH_BG_PTR(p); be=(back_edges*)((word)back_ptr&~FLAG_MANY); } be->flags|=RETAIN; be->height=p_height; be->height_gc_no=(unsigned short)GC_gc_no; } if (p_height > GC_max_height){ GC_max_height=p_height; GC_deepest_obj=p_deepest_obj; } } } STATIC word GC_max_max_height=0; GC_INNER void GC_traverse_back_graph(void) { GC_ASSERT(I_HOLD_LOCK()); GC_max_height=0; GC_apply_to_each_object(update_max_height); if (0!=GC_deepest_obj) GC_set_mark_bit(GC_deepest_obj); } void GC_print_back_graph_stats(void) { GC_ASSERT(I_HOLD_LOCK()); GC_printf("Maximum backwards height of reachable objects at GC %lu is %lu\n", (unsigned long)GC_gc_no,(unsigned long)GC_max_height); if (GC_max_height > GC_max_max_height){ ptr_t obj=GC_deepest_obj; GC_max_max_height=GC_max_height; UNLOCK(); GC_err_printf( "The following unreachable object is last in a longest chain " "of unreachable objects:\n"); GC_print_heap_obj(obj); LOCK(); } GC_COND_LOG_PRINTF("Needed max total of %d back-edge structs\n", GC_n_back_edge_structs); GC_apply_to_each_object(reset_back_edge); GC_deepest_obj=0; } #endif STATIC word*GC_old_normal_bl=NULL; STATIC word*GC_incomplete_normal_bl=NULL; STATIC word*GC_old_stack_bl=NULL; STATIC word*GC_incomplete_stack_bl=NULL; STATIC word GC_total_stack_black_listed=0; GC_INNER word GC_black_list_spacing=MINHINCR*HBLKSIZE; STATIC void GC_clear_bl(word*); GC_INNER void GC_default_print_heap_obj_proc(ptr_t p) { ptr_t base=(ptr_t)GC_base(p); int kind=HDR(base)->hb_obj_kind; GC_err_printf("object at %p of appr. %lu bytes (%s)\n", (void*)base,(unsigned long)GC_size(base), kind==PTRFREE?"atomic": IS_UNCOLLECTABLE(kind)?"uncollectable":"composite"); } GC_INNER void (*GC_print_heap_obj)(ptr_t p)=GC_default_print_heap_obj_proc; #ifdef PRINT_BLACK_LIST STATIC void GC_print_blacklisted_ptr(word p,ptr_t source, const char*kind_str) { ptr_t base=(ptr_t)GC_base(source); if (0==base){ GC_err_printf("Black listing (%s)%p referenced from %p in %s\n", kind_str,(void*)p,(void*)source, NULL!=source?"root set":"register"); } else { GC_err_printf("Black listing (%s)%p referenced from %p in" " object at %p of appr. %lu bytes\n", kind_str,(void*)p,(void*)source, (void*)base,(unsigned long)GC_size(base)); } } #endif GC_INNER void GC_bl_init_no_interiors(void) { if (GC_incomplete_normal_bl==0){ GC_old_normal_bl=(word*)GC_scratch_alloc(sizeof(page_hash_table)); GC_incomplete_normal_bl=(word*)GC_scratch_alloc( sizeof(page_hash_table)); if (GC_old_normal_bl==0||GC_incomplete_normal_bl==0){ GC_err_printf("Insufficient memory for black list\n"); EXIT(); } GC_clear_bl(GC_old_normal_bl); GC_clear_bl(GC_incomplete_normal_bl); } } GC_INNER void GC_bl_init(void) { if (!GC_all_interior_pointers){ GC_bl_init_no_interiors(); } GC_ASSERT(NULL==GC_old_stack_bl&&NULL==GC_incomplete_stack_bl); GC_old_stack_bl=(word*)GC_scratch_alloc(sizeof(page_hash_table)); GC_incomplete_stack_bl=(word*)GC_scratch_alloc(sizeof(page_hash_table)); if (GC_old_stack_bl==0||GC_incomplete_stack_bl==0){ GC_err_printf("Insufficient memory for black list\n"); EXIT(); } GC_clear_bl(GC_old_stack_bl); GC_clear_bl(GC_incomplete_stack_bl); } STATIC void GC_clear_bl(word*doomed) { BZERO(doomed,sizeof(page_hash_table)); } STATIC void GC_copy_bl(word*old,word*dest) { BCOPY(old,dest,sizeof(page_hash_table)); } static word total_stack_black_listed(void); GC_INNER void GC_promote_black_lists(void) { word*very_old_normal_bl=GC_old_normal_bl; word*very_old_stack_bl=GC_old_stack_bl; GC_old_normal_bl=GC_incomplete_normal_bl; GC_old_stack_bl=GC_incomplete_stack_bl; if (!GC_all_interior_pointers){ GC_clear_bl(very_old_normal_bl); } GC_clear_bl(very_old_stack_bl); GC_incomplete_normal_bl=very_old_normal_bl; GC_incomplete_stack_bl=very_old_stack_bl; GC_total_stack_black_listed=total_stack_black_listed(); GC_VERBOSE_LOG_PRINTF( "%lu bytes in heap blacklisted for interior pointers\n", (unsigned long)GC_total_stack_black_listed); if (GC_total_stack_black_listed!=0){ GC_black_list_spacing= HBLKSIZE*(GC_heapsize/GC_total_stack_black_listed); } if (GC_black_list_spacing < 3*HBLKSIZE){ GC_black_list_spacing=3*HBLKSIZE; } if (GC_black_list_spacing > MAXHINCR*HBLKSIZE){ GC_black_list_spacing=MAXHINCR*HBLKSIZE; } } GC_INNER void GC_unpromote_black_lists(void) { if (!GC_all_interior_pointers){ GC_copy_bl(GC_old_normal_bl,GC_incomplete_normal_bl); } GC_copy_bl(GC_old_stack_bl,GC_incomplete_stack_bl); } #if defined(PARALLEL_MARK)&&defined(THREAD_SANITIZER) #define backlist_set_pht_entry_from_index(db,index)set_pht_entry_from_index_concurrent(db,index) #else #define backlist_set_pht_entry_from_index(bl,index)set_pht_entry_from_index(bl,index) #endif #ifdef PRINT_BLACK_LIST GC_INNER void GC_add_to_black_list_normal(word p,ptr_t source) #else GC_INNER void GC_add_to_black_list_normal(word p) #endif { if (GC_modws_valid_offsets[p&(sizeof(word)-1)]){ word index=PHT_HASH((word)p); if (HDR(p)==0||get_pht_entry_from_index(GC_old_normal_bl,index)){ #ifdef PRINT_BLACK_LIST if (!get_pht_entry_from_index(GC_incomplete_normal_bl,index)){ GC_print_blacklisted_ptr(p,source,"normal"); } #endif backlist_set_pht_entry_from_index(GC_incomplete_normal_bl,index); } } } #ifdef PRINT_BLACK_LIST GC_INNER void GC_add_to_black_list_stack(word p,ptr_t source) #else GC_INNER void GC_add_to_black_list_stack(word p) #endif { word index=PHT_HASH((word)p); if (HDR(p)==0||get_pht_entry_from_index(GC_old_stack_bl,index)){ #ifdef PRINT_BLACK_LIST if (!get_pht_entry_from_index(GC_incomplete_stack_bl,index)){ GC_print_blacklisted_ptr(p,source,"stack"); } #endif backlist_set_pht_entry_from_index(GC_incomplete_stack_bl,index); } } struct hblk*GC_is_black_listed(struct hblk*h,word len) { word index=PHT_HASH((word)h); word i; word nblocks; if (!GC_all_interior_pointers &&(get_pht_entry_from_index(GC_old_normal_bl,index) ||get_pht_entry_from_index(GC_incomplete_normal_bl,index))){ return (h+1); } nblocks=divHBLKSZ(len); for (i=0;;){ if (GC_old_stack_bl[divWORDSZ(index)]==0 &&GC_incomplete_stack_bl[divWORDSZ(index)]==0){ i+=WORDSZ - modWORDSZ(index); } else { if (get_pht_entry_from_index(GC_old_stack_bl,index) ||get_pht_entry_from_index(GC_incomplete_stack_bl,index)){ return(h+i+1); } i++; } if (i>=nblocks)break; index=PHT_HASH((word)(h+i)); } return(0); } STATIC word GC_number_stack_black_listed(struct hblk*start, struct hblk*endp1) { struct hblk*h; word result=0; for (h=start;(word)h < (word)endp1;h++){ word index=PHT_HASH((word)h); if (get_pht_entry_from_index(GC_old_stack_bl,index))result++; } return(result); } static word total_stack_black_listed(void) { unsigned i; word total=0; for (i=0;i < GC_n_heap_sects;i++){ struct hblk*start=(struct hblk*)GC_heap_sects[i].hs_start; struct hblk*endp1=start+divHBLKSZ(GC_heap_sects[i].hs_bytes); total+=GC_number_stack_black_listed(start,endp1); } return(total*HBLKSIZE); } #ifdef CHECKSUMS #define NSUMS 10000 #define OFFSET 0x10000 typedef struct { GC_bool new_valid; word old_sum; word new_sum; struct hblk*block; } page_entry; page_entry GC_sums[NSUMS]; STATIC word GC_faulted[NSUMS]={ 0 }; STATIC size_t GC_n_faulted=0; #if defined(MPROTECT_VDB)&&!defined(DARWIN) void GC_record_fault(struct hblk*h) { word page=(word)h&~(GC_page_size - 1); GC_ASSERT(GC_page_size!=0); if (GC_n_faulted>=NSUMS)ABORT("write fault log overflowed"); GC_faulted[GC_n_faulted++]=page; } #endif STATIC GC_bool GC_was_faulted(struct hblk*h) { size_t i; word page=(word)h&~(GC_page_size - 1); for (i=0;i < GC_n_faulted;++i){ if (GC_faulted[i]==page)return TRUE; } return FALSE; } STATIC word GC_checksum(struct hblk*h) { word*p=(word*)h; word*lim=(word*)(h+1); word result=0; while ((word)p < (word)lim){ result+=*p++; } return(result|0x80000000); } int GC_n_dirty_errors=0; int GC_n_faulted_dirty_errors=0; unsigned long GC_n_clean=0; unsigned long GC_n_dirty=0; STATIC void GC_update_check_page(struct hblk*h,int index) { page_entry*pe=GC_sums+index; hdr*hhdr=HDR(h); struct hblk*b; if (pe->block!=0&&pe->block!=h+OFFSET)ABORT("goofed"); pe->old_sum=pe->new_sum; pe->new_sum=GC_checksum(h); #if!defined(MSWIN32)&&!defined(MSWINCE) if (pe->new_sum!=0x80000000&&!GC_page_was_ever_dirty(h)){ GC_err_printf("GC_page_was_ever_dirty(%p)is wrong\n",(void*)h); } #endif if (GC_page_was_dirty(h)){ GC_n_dirty++; } else { GC_n_clean++; } b=h; while (IS_FORWARDING_ADDR_OR_NIL(hhdr)&&hhdr!=0){ b-=(word)hhdr; hhdr=HDR(b); } if (pe->new_valid &&hhdr!=0&&hhdr->hb_descr!=0 &&pe->old_sum!=pe->new_sum){ if (!GC_page_was_dirty(h)||!GC_page_was_ever_dirty(h)){ GC_bool was_faulted=GC_was_faulted(h); GC_n_dirty_errors++; if (was_faulted)GC_n_faulted_dirty_errors++; } } pe->new_valid=TRUE; pe->block=h+OFFSET; } word GC_bytes_in_used_blocks=0; STATIC void GC_add_block(struct hblk*h,word dummy GC_ATTR_UNUSED) { hdr*hhdr=HDR(h); GC_bytes_in_used_blocks+=(hhdr->hb_sz+HBLKSIZE-1)&~(HBLKSIZE-1); } STATIC void GC_check_blocks(void) { word bytes_in_free_blocks=GC_large_free_bytes; GC_bytes_in_used_blocks=0; GC_apply_to_all_blocks(GC_add_block,(word)0); GC_COND_LOG_PRINTF("GC_bytes_in_used_blocks=%lu," " bytes_in_free_blocks=%lu,heapsize=%lu\n", (unsigned long)GC_bytes_in_used_blocks, (unsigned long)bytes_in_free_blocks, (unsigned long)GC_heapsize); if (GC_bytes_in_used_blocks+bytes_in_free_blocks!=GC_heapsize){ GC_err_printf("LOST SOME BLOCKS!!\n"); } } void GC_check_dirty(void) { int index; unsigned i; struct hblk*h; ptr_t start; GC_check_blocks(); GC_n_dirty_errors=0; GC_n_faulted_dirty_errors=0; GC_n_clean=0; GC_n_dirty=0; index=0; for (i=0;i < GC_n_heap_sects;i++){ start=GC_heap_sects[i].hs_start; for (h=(struct hblk*)start; (word)h < (word)(start+GC_heap_sects[i].hs_bytes);h++){ GC_update_check_page(h,index); index++; if (index>=NSUMS)goto out; } } out: GC_COND_LOG_PRINTF("Checked %lu clean and %lu dirty pages\n", GC_n_clean,GC_n_dirty); if (GC_n_dirty_errors > 0){ GC_err_printf("Found %d dirty bit errors (%d were faulted)\n", GC_n_dirty_errors,GC_n_faulted_dirty_errors); } for (i=0;i < GC_n_faulted;++i){ GC_faulted[i]=0; } GC_n_faulted=0; } #endif #ifndef GC_PMARK_H #define GC_PMARK_H #if defined(HAVE_CONFIG_H)&&!defined(GC_PRIVATE_H) #endif #ifndef GC_BUILD #define GC_BUILD #endif #if (defined(__linux__)||defined(__GLIBC__)||defined(__GNU__))&&!defined(_GNU_SOURCE)&&defined(GC_PTHREADS)&&!defined(GC_NO_PTHREAD_SIGMASK) #define _GNU_SOURCE 1 #endif #if defined(KEEP_BACK_PTRS)||defined(PRINT_BLACK_LIST) #endif EXTERN_C_BEGIN #ifndef MARK_DESCR_OFFSET #define MARK_DESCR_OFFSET sizeof(word) #endif #define BITMAP_BITS (WORDSZ - GC_DS_TAG_BITS) #define PROC(descr)(GC_mark_procs[((descr)>>GC_DS_TAG_BITS)&(GC_MAX_MARK_PROCS-1)]) #define ENV(descr)((descr)>>(GC_DS_TAG_BITS+GC_LOG_MAX_MARK_PROCS)) #define MAX_ENV (((word)1<<(WORDSZ - GC_DS_TAG_BITS - GC_LOG_MAX_MARK_PROCS))- 1) GC_EXTERN unsigned GC_n_mark_procs; #define GC_MARK_STACK_DISCARDS (INITIAL_MARK_STACK_SIZE/8) #ifdef PARALLEL_MARK #endif GC_INNER mse*GC_signal_mark_stack_overflow(mse*msp); GC_INLINE mse*GC_push_obj(ptr_t obj,hdr*hhdr,mse*mark_stack_top, mse*mark_stack_limit) { word descr=hhdr->hb_descr; GC_ASSERT(!HBLK_IS_FREE(hhdr)); if (descr!=0){ mark_stack_top++; if ((word)mark_stack_top>=(word)mark_stack_limit){ mark_stack_top=GC_signal_mark_stack_overflow(mark_stack_top); } mark_stack_top->mse_start=obj; mark_stack_top->mse_descr.w=descr; } return mark_stack_top; } #define PUSH_CONTENTS(current,mark_stack_top,mark_stack_limit,source)do { hdr*my_hhdr;HC_GET_HDR(current,my_hhdr,source);mark_stack_top=GC_push_contents_hdr(current,mark_stack_top,mark_stack_limit,source,my_hhdr,TRUE);} while (0) #ifdef USE_MARK_BYTES #if defined(PARALLEL_MARK)&&defined(AO_HAVE_char_store)&&!defined(BASE_ATOMIC_OPS_EMULATED) #define SET_MARK_BIT_EXIT_IF_SET(hhdr,bit_no){ volatile unsigned char*mark_byte_addr=(unsigned char*)(hhdr)->hb_marks+(bit_no);if (AO_char_load(mark_byte_addr)!=0)break;AO_char_store(mark_byte_addr,1);} #else #define SET_MARK_BIT_EXIT_IF_SET(hhdr,bit_no){ char*mark_byte_addr=(char*)(hhdr)->hb_marks+(bit_no);if (*mark_byte_addr!=0)break;*mark_byte_addr=1;} #endif #else #ifdef PARALLEL_MARK #ifdef THREAD_SANITIZER #define OR_WORD_EXIT_IF_SET(addr,bits){ if (!((word)AO_load((volatile AO_t*)(addr))&(bits))){ AO_or((volatile AO_t*)(addr),(AO_t)(bits));} else { break;} } #else #define OR_WORD_EXIT_IF_SET(addr,bits){ if (!(*(addr)&(bits))){ AO_or((volatile AO_t*)(addr),(AO_t)(bits));} else { break;} } #endif #else #define OR_WORD_EXIT_IF_SET(addr,bits){ word old=*(addr);word my_bits=(bits);if ((old&my_bits)!=0)break;*(addr)=old|my_bits;} #endif #define SET_MARK_BIT_EXIT_IF_SET(hhdr,bit_no){ word*mark_word_addr=(hhdr)->hb_marks+divWORDSZ(bit_no);OR_WORD_EXIT_IF_SET(mark_word_addr,(word)1<hb_n_marks,AO_load(&hhdr->hb_n_marks)+1) #else #define INCR_MARKS(hhdr)(void)(++hhdr->hb_n_marks) #endif #ifdef ENABLE_TRACE #define TRACE(source,cmd)if (GC_trace_addr!=0&&(ptr_t)(source)==GC_trace_addr)cmd #define TRACE_TARGET(target,cmd)if (GC_trace_addr!=0&&(target)==*(ptr_t*)GC_trace_addr)cmd #else #define TRACE(source,cmd) #define TRACE_TARGET(source,cmd) #endif #if defined(I386)&&defined(__GNUC__)&&!defined(NACL) #define LONG_MULT(hprod,lprod,x,y)do { __asm__ __volatile__("mull %2":"=a"(lprod),"=d"(hprod):"g"(y),"0"(x));} while (0) #else #if defined(__int64)&&!defined(__GNUC__)&&!defined(CPPCHECK) #define ULONG_MULT_T unsigned __int64 #else #define ULONG_MULT_T unsigned long long #endif #define LONG_MULT(hprod,lprod,x,y)do { ULONG_MULT_T prod=(ULONG_MULT_T)(x)*(ULONG_MULT_T)(y);GC_STATIC_ASSERT(sizeof(x)+sizeof(y)<=sizeof(prod));hprod=prod>>32;lprod=(unsigned32)prod;} while (0) #endif GC_INLINE mse*GC_push_contents_hdr(ptr_t current,mse*mark_stack_top, mse*mark_stack_limit,ptr_t source, hdr*hhdr,GC_bool do_offset_check) { do { size_t displ=HBLKDISPL(current); ptr_t base=current; #ifdef MARK_BIT_PER_GRANULE size_t gran_displ=BYTES_TO_GRANULES(displ); size_t gran_offset=hhdr->hb_map[gran_displ]; size_t byte_offset=displ&(GRANULE_BYTES - 1); if (EXPECT((gran_offset|byte_offset)!=0,FALSE)) #else unsigned32 gran_displ; unsigned32 inv_sz=hhdr->hb_inv_sz; #endif { #ifdef MARK_BIT_PER_GRANULE if ((hhdr->hb_flags&LARGE_BLOCK)!=0) #else if (EXPECT(inv_sz==LARGE_INV_SZ,FALSE)) #endif { size_t obj_displ; base=(ptr_t)hhdr->hb_block; obj_displ=current - base; if (obj_displ!=displ){ GC_ASSERT(obj_displ < hhdr->hb_sz); } else if (do_offset_check&&!GC_valid_offsets[obj_displ]){ GC_ADD_TO_BLACK_LIST_NORMAL(current,source); break; } GC_ASSERT(hhdr->hb_sz > HBLKSIZE ||hhdr->hb_block==HBLKPTR(current)); GC_ASSERT((word)hhdr->hb_block<=(word)current); gran_displ=0; } else { #ifdef MARK_BIT_PER_GRANULE size_t obj_displ=GRANULES_TO_BYTES(gran_offset)+byte_offset; #else unsigned32 low_prod; LONG_MULT(gran_displ,low_prod,(unsigned32)displ,inv_sz); if ((low_prod>>16)!=0) #endif { #if defined(MARK_BIT_PER_OBJ)&&!defined(MARK_BIT_PER_GRANULE) size_t obj_displ; GC_STATIC_ASSERT(HBLKSIZE<=(1<<15)); obj_displ=(((low_prod>>16)+1)*(size_t)hhdr->hb_sz)>>16; #endif if (do_offset_check&&!GC_valid_offsets[obj_displ]){ GC_ADD_TO_BLACK_LIST_NORMAL(current,source); break; } #ifdef MARK_BIT_PER_GRANULE gran_displ-=gran_offset; #endif base-=obj_displ; } } } #ifdef MARK_BIT_PER_GRANULE GC_ASSERT(hhdr==GC_find_header(base)); GC_ASSERT(gran_displ % BYTES_TO_GRANULES(hhdr->hb_sz)==0); #else GC_ASSERT(gran_displ<=HBLK_OBJS(hhdr->hb_sz)); #endif TRACE(source,GC_log_printf("GC #%u:passed validity tests\n", (unsigned)GC_gc_no)); SET_MARK_BIT_EXIT_IF_SET(hhdr,gran_displ); TRACE(source,GC_log_printf("GC #%u:previously unmarked\n", (unsigned)GC_gc_no)); TRACE_TARGET(base,GC_log_printf("GC #%u:marking %p from %p instead\n", (unsigned)GC_gc_no,(void*)base, (void*)source)); INCR_MARKS(hhdr); GC_STORE_BACK_PTR(source,base); mark_stack_top=GC_push_obj(base,hhdr,mark_stack_top, mark_stack_limit); } while (0); return mark_stack_top; } #if defined(PRINT_BLACK_LIST)||defined(KEEP_BACK_PTRS) #define PUSH_ONE_CHECKED_STACK(p,source)GC_mark_and_push_stack((ptr_t)(p),(ptr_t)(source)) #else #define PUSH_ONE_CHECKED_STACK(p,source)GC_mark_and_push_stack((ptr_t)(p)) #endif #ifdef NEED_FIXUP_POINTER #define GC_PUSH_ONE_STACK(p,source)do { if ((word)(p)>=(word)GC_least_plausible_heap_addr&&(word)(p)< (word)GC_greatest_plausible_heap_addr){ PUSH_ONE_CHECKED_STACK(p,source);} FIXUP_POINTER(p);if ((word)(p)>=(word)GC_least_plausible_heap_addr&&(word)(p)< (word)GC_greatest_plausible_heap_addr){ PUSH_ONE_CHECKED_STACK(p,source);} } while (0) #else #define GC_PUSH_ONE_STACK(p,source)do { if ((word)(p)>=(word)GC_least_plausible_heap_addr&&(word)(p)< (word)GC_greatest_plausible_heap_addr){ PUSH_ONE_CHECKED_STACK(p,source);} } while (0) #endif #define GC_PUSH_ONE_HEAP(p,source,mark_stack_top)do { FIXUP_POINTER(p);if ((word)(p)>=(word)GC_least_plausible_heap_addr&&(word)(p)< (word)GC_greatest_plausible_heap_addr)mark_stack_top=GC_mark_and_push((void*)(p),mark_stack_top,GC_mark_stack_limit,(void**)(source));} while (0) GC_INNER mse*GC_mark_from(mse*top,mse*bottom,mse*limit); #define MARK_FROM_MARK_STACK()GC_mark_stack_top=GC_mark_from(GC_mark_stack_top,GC_mark_stack,GC_mark_stack+GC_mark_stack_size); #define GC_mark_stack_empty()((word)GC_mark_stack_top < (word)GC_mark_stack) #define GC_MARK_FO(real_ptr,mark_proc)do { (*(mark_proc))(real_ptr);while (!GC_mark_stack_empty())MARK_FROM_MARK_STACK();if (GC_mark_state!=MS_NONE){ GC_set_mark_bit(real_ptr);while (!GC_mark_some((ptr_t)0)){ } } } while (0) #define MS_NONE 0 #define MS_PUSH_RESCUERS 1 #define MS_PUSH_UNCOLLECTABLE 2 #define MS_ROOTS_PUSHED 3 #define MS_PARTIALLY_INVALID 4 #define MS_INVALID 5 EXTERN_C_END #endif #ifdef GC_GCJ_SUPPORT #ifndef GC_GCJ_H #define GC_GCJ_H #ifndef GC_H #endif #ifdef __cplusplus extern "C" { #endif GC_API void GC_CALL GC_init_gcj_malloc(int, void*); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_gcj_malloc(size_t, void*); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_debug_gcj_malloc(size_t, void*, GC_EXTRA_PARAMS); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_gcj_malloc_ignore_off_page(size_t, void*); GC_API int GC_gcj_kind; GC_API int GC_gcj_debug_kind; #ifdef GC_DEBUG #define GC_GCJ_MALLOC(s,d)GC_debug_gcj_malloc(s,d,GC_EXTRAS) #define GC_GCJ_MALLOC_IGNORE_OFF_PAGE(s,d)GC_debug_gcj_malloc(s,d,GC_EXTRAS) #else #define GC_GCJ_MALLOC(s,d)GC_gcj_malloc(s,d) #define GC_GCJ_MALLOC_IGNORE_OFF_PAGE(s,d)GC_gcj_malloc_ignore_off_page(s,d) #endif #ifdef __cplusplus } #endif #endif int GC_gcj_kind=0; int GC_gcj_debug_kind=0; STATIC struct GC_ms_entry*GC_gcj_fake_mark_proc(word*addr GC_ATTR_UNUSED, struct GC_ms_entry*mark_stack_ptr, struct GC_ms_entry*mark_stack_limit GC_ATTR_UNUSED, word env GC_ATTR_UNUSED) { ABORT_RET("No client gcj mark proc is specified"); return mark_stack_ptr; } GC_API void GC_CALL GC_init_gcj_malloc(int mp_index, void*mp) { #ifndef GC_IGNORE_GCJ_INFO GC_bool ignore_gcj_info; #endif DCL_LOCK_STATE; if (mp==0) mp=(void*)(word)GC_gcj_fake_mark_proc; GC_init(); LOCK(); if (GC_gcjobjfreelist!=NULL){ UNLOCK(); return; } #ifdef GC_IGNORE_GCJ_INFO #define ignore_gcj_info TRUE #else ignore_gcj_info=(0!=GETENV("GC_IGNORE_GCJ_INFO")); #endif if (ignore_gcj_info){ GC_COND_LOG_PRINTF("Gcj-style type information is disabled!\n"); } GC_ASSERT(GC_mark_procs[mp_index]==(GC_mark_proc)0); GC_mark_procs[mp_index]=(GC_mark_proc)(word)mp; if ((unsigned)mp_index>=GC_n_mark_procs) ABORT("GC_init_gcj_malloc:bad index"); GC_gcjobjfreelist=(ptr_t*)GC_new_free_list_inner(); if (ignore_gcj_info){ GC_gcj_kind=GC_new_kind_inner((void**)GC_gcjobjfreelist, GC_DS_LENGTH, TRUE,TRUE); GC_gcj_debug_kind=GC_gcj_kind; } else { GC_gcj_kind=GC_new_kind_inner( (void**)GC_gcjobjfreelist, (((word)(-(signed_word)MARK_DESCR_OFFSET - GC_INDIR_PER_OBJ_BIAS)) |GC_DS_PER_OBJECT), FALSE,TRUE); GC_gcj_debug_kind=GC_new_kind_inner(GC_new_free_list_inner(), GC_MAKE_PROC(mp_index, 1), FALSE,TRUE); } UNLOCK(); #undef ignore_gcj_info } #define GENERAL_MALLOC_INNER(lb,k)GC_clear_stack(GC_generic_malloc_inner(lb,k)) #define GENERAL_MALLOC_INNER_IOP(lb,k)GC_clear_stack(GC_generic_malloc_inner_ignore_off_page(lb,k)) static void maybe_finalize(void) { static word last_finalized_no=0; DCL_LOCK_STATE; if (GC_gc_no==last_finalized_no|| !EXPECT(GC_is_initialized,TRUE))return; UNLOCK(); GC_INVOKE_FINALIZERS(); LOCK(); last_finalized_no=GC_gc_no; } #ifdef THREAD_LOCAL_ALLOC GC_INNER void*GC_core_gcj_malloc(size_t lb, void*ptr_to_struct_containing_descr) #else GC_API GC_ATTR_MALLOC void*GC_CALL GC_gcj_malloc(size_t lb, void*ptr_to_struct_containing_descr) #endif { ptr_t op; DCL_LOCK_STATE; GC_DBG_COLLECT_AT_MALLOC(lb); if(SMALL_OBJ(lb)){ word lg; LOCK(); lg=GC_size_map[lb]; op=GC_gcjobjfreelist[lg]; if(EXPECT(0==op,FALSE)){ maybe_finalize(); op=(ptr_t)GENERAL_MALLOC_INNER((word)lb,GC_gcj_kind); if (0==op){ GC_oom_func oom_fn=GC_oom_fn; UNLOCK(); return((*oom_fn)(lb)); } } else { GC_gcjobjfreelist[lg]=(ptr_t)obj_link(op); GC_bytes_allocd+=GRANULES_TO_BYTES((word)lg); } GC_ASSERT(((void**)op)[1]==0); } else { LOCK(); maybe_finalize(); op=(ptr_t)GENERAL_MALLOC_INNER((word)lb,GC_gcj_kind); if (0==op){ GC_oom_func oom_fn=GC_oom_fn; UNLOCK(); return((*oom_fn)(lb)); } } *(void**)op=ptr_to_struct_containing_descr; UNLOCK(); GC_dirty(op); REACHABLE_AFTER_DIRTY(ptr_to_struct_containing_descr); return (void*)op; } GC_API GC_ATTR_MALLOC void*GC_CALL GC_debug_gcj_malloc(size_t lb, void*ptr_to_struct_containing_descr,GC_EXTRA_PARAMS) { void*result; DCL_LOCK_STATE; LOCK(); maybe_finalize(); result=GC_generic_malloc_inner(SIZET_SAT_ADD(lb,DEBUG_BYTES), GC_gcj_debug_kind); if (result==0){ GC_oom_func oom_fn=GC_oom_fn; UNLOCK(); GC_err_printf("GC_debug_gcj_malloc(%lu,%p)returning NULL (%s:%d)\n", (unsigned long)lb,ptr_to_struct_containing_descr,s,i); return((*oom_fn)(lb)); } *((void**)((ptr_t)result+sizeof(oh)))=ptr_to_struct_containing_descr; if (!GC_debugging_started){ GC_start_debugging_inner(); } ADD_CALL_CHAIN(result,ra); result=GC_store_debug_info_inner(result,(word)lb,s,i); UNLOCK(); GC_dirty(result); REACHABLE_AFTER_DIRTY(ptr_to_struct_containing_descr); return result; } GC_API GC_ATTR_MALLOC void*GC_CALL GC_gcj_malloc_ignore_off_page(size_t lb, void*ptr_to_struct_containing_descr) { ptr_t op; DCL_LOCK_STATE; GC_DBG_COLLECT_AT_MALLOC(lb); if(SMALL_OBJ(lb)){ word lg; LOCK(); lg=GC_size_map[lb]; op=GC_gcjobjfreelist[lg]; if (EXPECT(0==op,FALSE)){ maybe_finalize(); op=(ptr_t)GENERAL_MALLOC_INNER_IOP(lb,GC_gcj_kind); if (0==op){ GC_oom_func oom_fn=GC_oom_fn; UNLOCK(); return((*oom_fn)(lb)); } } else { GC_gcjobjfreelist[lg]=(ptr_t)obj_link(op); GC_bytes_allocd+=GRANULES_TO_BYTES((word)lg); } } else { LOCK(); maybe_finalize(); op=(ptr_t)GENERAL_MALLOC_INNER_IOP(lb,GC_gcj_kind); if (0==op){ GC_oom_func oom_fn=GC_oom_fn; UNLOCK(); return((*oom_fn)(lb)); } } *(void**)op=ptr_to_struct_containing_descr; UNLOCK(); GC_dirty(op); REACHABLE_AFTER_DIRTY(ptr_to_struct_containing_descr); return (void*)op; } #endif GC_INNER hdr*GC_find_header(ptr_t h) { #ifdef HASH_TL hdr*result; GET_HDR(h,result); return(result); #else return(HDR_INNER(h)); #endif } GC_INNER hdr* #ifdef PRINT_BLACK_LIST GC_header_cache_miss(ptr_t p,hdr_cache_entry*hce,ptr_t source) #else GC_header_cache_miss(ptr_t p,hdr_cache_entry*hce) #endif { hdr*hhdr; HC_MISS(); GET_HDR(p,hhdr); if (IS_FORWARDING_ADDR_OR_NIL(hhdr)){ if (GC_all_interior_pointers){ if (hhdr!=0){ ptr_t current=p; current=(ptr_t)HBLKPTR(current); do { current=current - HBLKSIZE*(word)hhdr; hhdr=HDR(current); } while(IS_FORWARDING_ADDR_OR_NIL(hhdr)); if (hhdr->hb_flags&IGNORE_OFF_PAGE) return 0; if (HBLK_IS_FREE(hhdr) ||p - current>=(ptrdiff_t)(hhdr->hb_sz)){ GC_ADD_TO_BLACK_LIST_NORMAL(p,source); return 0; } } else { GC_ADD_TO_BLACK_LIST_NORMAL(p,source); } GC_ASSERT(hhdr==0||!HBLK_IS_FREE(hhdr)); return hhdr; } else { if (hhdr==0){ GC_ADD_TO_BLACK_LIST_NORMAL(p,source); } return 0; } } else { if (HBLK_IS_FREE(hhdr)){ GC_ADD_TO_BLACK_LIST_NORMAL(p,source); return 0; } else { hce->block_addr=(word)(p)>>LOG_HBLKSIZE; hce->hce_hdr=hhdr; return hhdr; } } } GC_INNER ptr_t GC_scratch_alloc(size_t bytes) { ptr_t result=GC_scratch_free_ptr; size_t bytes_to_get; bytes=ROUNDUP_GRANULE_SIZE(bytes); for (;;){ GC_scratch_free_ptr+=bytes; if ((word)GC_scratch_free_ptr<=(word)GC_scratch_end_ptr){ return result; } GC_ASSERT(GC_page_size!=0); if (bytes>=MINHINCR*HBLKSIZE){ bytes_to_get=ROUNDUP_PAGESIZE_IF_MMAP(bytes); result=(ptr_t)GET_MEM(bytes_to_get); GC_add_to_our_memory(result,bytes_to_get); GC_scratch_free_ptr-=bytes; if (result!=NULL){ GC_scratch_last_end_ptr=result+bytes; } return result; } bytes_to_get=ROUNDUP_PAGESIZE_IF_MMAP(MINHINCR*HBLKSIZE); result=(ptr_t)GET_MEM(bytes_to_get); GC_add_to_our_memory(result,bytes_to_get); if (NULL==result){ WARN("Out of memory - trying to allocate requested amount" " (%" WARN_PRIdPTR " bytes)...\n",(word)bytes); GC_scratch_free_ptr-=bytes; bytes_to_get=ROUNDUP_PAGESIZE_IF_MMAP(bytes); result=(ptr_t)GET_MEM(bytes_to_get); GC_add_to_our_memory(result,bytes_to_get); return result; } GC_scratch_free_ptr=result; GC_scratch_end_ptr=GC_scratch_free_ptr+bytes_to_get; GC_scratch_last_end_ptr=GC_scratch_end_ptr; } } static hdr*alloc_hdr(void) { hdr*result; if (NULL==GC_hdr_free_list){ result=(hdr*)GC_scratch_alloc(sizeof(hdr)); } else { result=GC_hdr_free_list; GC_hdr_free_list=(hdr*)result->hb_next; } return(result); } GC_INLINE void free_hdr(hdr*hhdr) { hhdr->hb_next=(struct hblk*)GC_hdr_free_list; GC_hdr_free_list=hhdr; } #ifdef COUNT_HDR_CACHE_HITS word GC_hdr_cache_hits=0; word GC_hdr_cache_misses=0; #endif GC_INNER void GC_init_headers(void) { unsigned i; GC_ASSERT(NULL==GC_all_nils); GC_all_nils=(bottom_index*)GC_scratch_alloc(sizeof(bottom_index)); if (GC_all_nils==NULL){ GC_err_printf("Insufficient memory for GC_all_nils\n"); EXIT(); } BZERO(GC_all_nils,sizeof(bottom_index)); for (i=0;i < TOP_SZ;i++){ GC_top_index[i]=GC_all_nils; } } static GC_bool get_index(word addr) { word hi=(word)(addr)>>(LOG_BOTTOM_SZ+LOG_HBLKSIZE); bottom_index*r; bottom_index*p; bottom_index**prev; bottom_index*pi; word i; GC_ASSERT(I_HOLD_LOCK()); #ifdef HASH_TL i=TL_HASH(hi); pi=p=GC_top_index[i]; while(p!=GC_all_nils){ if (p->key==hi)return(TRUE); p=p->hash_link; } #else if (GC_top_index[hi]!=GC_all_nils) return TRUE; i=hi; #endif r=(bottom_index*)GC_scratch_alloc(sizeof(bottom_index)); if (EXPECT(NULL==r,FALSE)) return FALSE; BZERO(r,sizeof(bottom_index)); r->key=hi; #ifdef HASH_TL r->hash_link=pi; #endif prev=&GC_all_bottom_indices; pi=0; while ((p=*prev)!=0&&p->key < hi){ pi=p; prev=&(p->asc_link); } r->desc_link=pi; if (0==p){ GC_all_bottom_indices_end=r; } else { p->desc_link=r; } r->asc_link=p; *prev=r; GC_top_index[i]=r; return(TRUE); } GC_INNER struct hblkhdr*GC_install_header(struct hblk*h) { hdr*result; if (!get_index((word)h))return(0); result=alloc_hdr(); if (result){ SET_HDR(h,result); #ifdef USE_MUNMAP result->hb_last_reclaimed=(unsigned short)GC_gc_no; #endif } return(result); } GC_INNER GC_bool GC_install_counts(struct hblk*h,size_t sz) { struct hblk*hbp; for (hbp=h;(word)hbp < (word)h+sz;hbp+=BOTTOM_SZ){ if (!get_index((word)hbp)) return FALSE; if ((word)hbp > GC_WORD_MAX - (word)BOTTOM_SZ*HBLKSIZE) break; } if (!get_index((word)h+sz - 1)) return FALSE; for (hbp=h+1;(word)hbp < (word)h+sz;hbp+=1){ word i=HBLK_PTR_DIFF(hbp,h); SET_HDR(hbp,(hdr*)(i > MAX_JUMP?MAX_JUMP:i)); } return TRUE; } GC_INNER void GC_remove_header(struct hblk*h) { hdr**ha; GET_HDR_ADDR(h,ha); free_hdr(*ha); *ha=0; } GC_INNER void GC_remove_counts(struct hblk*h,size_t sz) { struct hblk*hbp; for (hbp=h+1;(word)hbp < (word)h+sz;hbp+=1){ SET_HDR(hbp,0); } } void GC_apply_to_all_blocks(void (*fn)(struct hblk*h,word client_data), word client_data) { signed_word j; bottom_index*index_p; for (index_p=GC_all_bottom_indices;index_p!=0; index_p=index_p->asc_link){ for (j=BOTTOM_SZ-1;j>=0;){ if (!IS_FORWARDING_ADDR_OR_NIL(index_p->index[j])){ if (!HBLK_IS_FREE(index_p->index[j])){ (*fn)(((struct hblk*) (((index_p->key<index[j]==0){ j--; } else { j-=(signed_word)(index_p->index[j]); } } } } GC_INNER struct hblk*GC_next_block(struct hblk*h,GC_bool allow_free) { REGISTER bottom_index*bi; REGISTER word j=((word)h>>LOG_HBLKSIZE)&(BOTTOM_SZ-1); GC_ASSERT(I_HOLD_LOCK()); GET_BI(h,bi); if (bi==GC_all_nils){ REGISTER word hi=(word)h>>(LOG_BOTTOM_SZ+LOG_HBLKSIZE); bi=GC_all_bottom_indices; while (bi!=0&&bi->key < hi)bi=bi->asc_link; j=0; } while (bi!=0){ while (j < BOTTOM_SZ){ hdr*hhdr=bi->index[j]; if (IS_FORWARDING_ADDR_OR_NIL(hhdr)){ j++; } else { if (allow_free||!HBLK_IS_FREE(hhdr)){ return ((struct hblk*) (((bi->key<hb_sz); } } } j=0; bi=bi->asc_link; } return(0); } GC_INNER struct hblk*GC_prev_block(struct hblk*h) { bottom_index*bi; signed_word j=((word)h>>LOG_HBLKSIZE)&(BOTTOM_SZ-1); GC_ASSERT(I_HOLD_LOCK()); GET_BI(h,bi); if (bi==GC_all_nils){ word hi=(word)h>>(LOG_BOTTOM_SZ+LOG_HBLKSIZE); bi=GC_all_bottom_indices_end; while (bi!=0&&bi->key > hi)bi=bi->desc_link; j=BOTTOM_SZ - 1; } while(bi!=0){ while (j>=0){ hdr*hhdr=bi->index[j]; if (0==hhdr){ --j; } else if (IS_FORWARDING_ADDR_OR_NIL(hhdr)){ j-=(signed_word)hhdr; } else { return((struct hblk*) (((bi->key<desc_link; } return(0); } #include #ifndef SMALL_CONFIG STATIC ptr_t GC_build_fl_clear2(struct hblk*h,ptr_t ofl) { word*p=(word*)(h->hb_body); word*lim=(word*)(h+1); p[0]=(word)ofl; p[1]=0; p[2]=(word)p; p[3]=0; p+=4; for (;(word)p < (word)lim;p+=4){ p[0]=(word)(p-2); p[1]=0; p[2]=(word)p; p[3]=0; }; return((ptr_t)(p-2)); } STATIC ptr_t GC_build_fl_clear4(struct hblk*h,ptr_t ofl) { word*p=(word*)(h->hb_body); word*lim=(word*)(h+1); p[0]=(word)ofl; p[1]=0; p[2]=0; p[3]=0; p+=4; for (;(word)p < (word)lim;p+=4){ GC_PREFETCH_FOR_WRITE((ptr_t)(p+64)); p[0]=(word)(p-4); p[1]=0; CLEAR_DOUBLE(p+2); }; return((ptr_t)(p-4)); } STATIC ptr_t GC_build_fl2(struct hblk*h,ptr_t ofl) { word*p=(word*)(h->hb_body); word*lim=(word*)(h+1); p[0]=(word)ofl; p[2]=(word)p; p+=4; for (;(word)p < (word)lim;p+=4){ p[0]=(word)(p-2); p[2]=(word)p; }; return((ptr_t)(p-2)); } STATIC ptr_t GC_build_fl4(struct hblk*h,ptr_t ofl) { word*p=(word*)(h->hb_body); word*lim=(word*)(h+1); p[0]=(word)ofl; p[4]=(word)p; p+=8; for (;(word)p < (word)lim;p+=8){ GC_PREFETCH_FOR_WRITE((ptr_t)(p+64)); p[0]=(word)(p-4); p[4]=(word)p; }; return((ptr_t)(p-4)); } #endif GC_INNER ptr_t GC_build_fl(struct hblk*h,size_t sz,GC_bool clear, ptr_t list) { word*p,*prev; word*last_object; GC_PREFETCH_FOR_WRITE((ptr_t)h); GC_PREFETCH_FOR_WRITE((ptr_t)h+128); GC_PREFETCH_FOR_WRITE((ptr_t)h+256); GC_PREFETCH_FOR_WRITE((ptr_t)h+378); #ifndef SMALL_CONFIG switch (sz){ case 2:if (clear){ return GC_build_fl_clear2(h,list); } else { return GC_build_fl2(h,list); } case 4:if (clear){ return GC_build_fl_clear4(h,list); } else { return GC_build_fl4(h,list); } default: break; } #endif if (clear)BZERO(h,HBLKSIZE); p=(word*)(h->hb_body)+sz; prev=(word*)(h->hb_body); last_object=(word*)((char*)h+HBLKSIZE); last_object-=sz; while ((word)p<=(word)last_object){ obj_link(p)=(ptr_t)prev; prev=p; p+=sz; } p-=sz; *(ptr_t*)h=list; return ((ptr_t)p); } GC_INNER void GC_new_hblk(size_t gran,int kind) { struct hblk*h; GC_bool clear=GC_obj_kinds[kind].ok_init; GC_STATIC_ASSERT((sizeof (struct hblk))==HBLKSIZE); if (GC_debugging_started)clear=TRUE; h=GC_allochblk(GRANULES_TO_BYTES(gran),kind,0); if (h==0)return; if (IS_UNCOLLECTABLE(kind))GC_set_hdr_marks(HDR(h)); GC_obj_kinds[kind].ok_freelist[gran]= GC_build_fl(h,GRANULES_TO_WORDS(gran),clear, (ptr_t)GC_obj_kinds[kind].ok_freelist[gran]); } GC_API void GC_CALL GC_register_displacement(size_t offset) { DCL_LOCK_STATE; LOCK(); GC_register_displacement_inner(offset); UNLOCK(); } GC_INNER void GC_register_displacement_inner(size_t offset) { GC_ASSERT(I_HOLD_LOCK()); if (offset>=VALID_OFFSET_SZ){ ABORT("Bad argument to GC_register_displacement"); } if (!GC_valid_offsets[offset]){ GC_valid_offsets[offset]=TRUE; GC_modws_valid_offsets[offset % sizeof(word)]=TRUE; } } #ifdef MARK_BIT_PER_GRANULE GC_INNER GC_bool GC_add_map_entry(size_t granules) { unsigned displ; unsigned short*new_map; if (granules > BYTES_TO_GRANULES(MAXOBJBYTES))granules=0; if (GC_obj_map[granules]!=0){ return(TRUE); } new_map=(unsigned short*)GC_scratch_alloc(MAP_LEN*sizeof(short)); if (new_map==0)return(FALSE); GC_COND_LOG_PRINTF( "Adding block map for size of %u granules (%u bytes)\n", (unsigned)granules,(unsigned)GRANULES_TO_BYTES(granules)); if (granules==0){ for (displ=0;displ < BYTES_TO_GRANULES(HBLKSIZE);displ++){ new_map[displ]=1; } } else { for (displ=0;displ < BYTES_TO_GRANULES(HBLKSIZE);displ++){ new_map[displ]=(unsigned short)(displ % granules); } } GC_obj_map[granules]=new_map; return(TRUE); } #endif GC_INNER void GC_initialize_offsets(void) { unsigned i; if (GC_all_interior_pointers){ for (i=0;i < VALID_OFFSET_SZ;++i) GC_valid_offsets[i]=TRUE; } else { BZERO(GC_valid_offsets,sizeof(GC_valid_offsets)); for (i=0;i < sizeof(word);++i) GC_modws_valid_offsets[i]=FALSE; } } STATIC void GC_CALLBACK GC_default_same_obj_print_proc(void*p,void*q) { ABORT_ARG2("GC_same_obj test failed", ":%p and %p are not in the same object",p,q); } void (GC_CALLBACK*GC_same_obj_print_proc)(void*,void*) =GC_default_same_obj_print_proc; GC_API void*GC_CALL GC_same_obj(void*p,void*q) { struct hblk*h; hdr*hhdr; ptr_t base,limit; word sz; if (!EXPECT(GC_is_initialized,TRUE))GC_init(); hhdr=HDR((word)p); if (hhdr==0){ if (divHBLKSZ((word)p)!=divHBLKSZ((word)q) &&HDR((word)q)!=0){ goto fail; } return(p); } if (IS_FORWARDING_ADDR_OR_NIL(hhdr)){ h=HBLKPTR(p)- (word)hhdr; hhdr=HDR(h); while (IS_FORWARDING_ADDR_OR_NIL(hhdr)){ h=FORWARDED_ADDR(h,hhdr); hhdr=HDR(h); } limit=(ptr_t)h+hhdr->hb_sz; if ((word)p>=(word)limit||(word)q>=(word)limit ||(word)q < (word)h){ goto fail; } return(p); } sz=hhdr->hb_sz; if (sz > MAXOBJBYTES){ base=(ptr_t)HBLKPTR(p); limit=base+sz; if ((word)p>=(word)limit){ goto fail; } } else { size_t offset; size_t pdispl=HBLKDISPL(p); offset=pdispl % sz; if (HBLKPTR(p)!=HBLKPTR(q))goto fail; base=(ptr_t)p - offset; limit=base+sz; } if ((word)q>=(word)limit||(word)q < (word)base){ goto fail; } return(p); fail: (*GC_same_obj_print_proc)((ptr_t)p,(ptr_t)q); return(p); } STATIC void GC_CALLBACK GC_default_is_valid_displacement_print_proc (void*p) { ABORT_ARG1("GC_is_valid_displacement test failed",":%p not valid",p); } void (GC_CALLBACK*GC_is_valid_displacement_print_proc)(void*)= GC_default_is_valid_displacement_print_proc; GC_API void*GC_CALL GC_is_valid_displacement(void*p) { hdr*hhdr; word pdispl; word offset; struct hblk*h; word sz; if (!EXPECT(GC_is_initialized,TRUE))GC_init(); hhdr=HDR((word)p); if (hhdr==0)return(p); h=HBLKPTR(p); if (GC_all_interior_pointers){ while (IS_FORWARDING_ADDR_OR_NIL(hhdr)){ h=FORWARDED_ADDR(h,hhdr); hhdr=HDR(h); } } else if (IS_FORWARDING_ADDR_OR_NIL(hhdr)){ goto fail; } sz=hhdr->hb_sz; pdispl=HBLKDISPL(p); offset=pdispl % sz; if ((sz > MAXOBJBYTES&&(word)p>=(word)h+sz) ||!GC_valid_offsets[offset] ||((word)p+(sz - offset)> (word)(h+1) &&!IS_FORWARDING_ADDR_OR_NIL(HDR(h+1)))){ goto fail; } return(p); fail: (*GC_is_valid_displacement_print_proc)((ptr_t)p); return(p); } STATIC void GC_CALLBACK GC_default_is_visible_print_proc(void*p) { ABORT_ARG1("GC_is_visible test failed",":%p not GC-visible",p); } void (GC_CALLBACK*GC_is_visible_print_proc)(void*p)= GC_default_is_visible_print_proc; #ifndef THREADS STATIC GC_bool GC_on_stack(void*p) { #ifdef STACK_GROWS_DOWN if ((word)p>=(word)GC_approx_sp() &&(word)p < (word)GC_stackbottom){ return(TRUE); } #else if ((word)p<=(word)GC_approx_sp() &&(word)p > (word)GC_stackbottom){ return(TRUE); } #endif return(FALSE); } #endif GC_API void*GC_CALL GC_is_visible(void*p) { hdr*hhdr; if ((word)p&(ALIGNMENT - 1))goto fail; if (!EXPECT(GC_is_initialized,TRUE))GC_init(); #ifdef THREADS hhdr=HDR((word)p); if (hhdr!=0&&GC_base(p)==0){ goto fail; } else { return(p); } #else if (GC_on_stack(p))return(p); hhdr=HDR((word)p); if (hhdr==0){ if (GC_is_static_root(p))return(p); #if defined(DYNAMIC_LOADING)||defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32)||defined(PCR) GC_register_dynamic_libraries(); if (GC_is_static_root(p)) return(p); #endif goto fail; } else { word descr; ptr_t base=(ptr_t)GC_base(p); if (NULL==base)goto fail; if (HBLKPTR(base)!=HBLKPTR(p)) hhdr=HDR(base); descr=hhdr->hb_descr; retry: switch(descr&GC_DS_TAGS){ case GC_DS_LENGTH: if ((word)p - (word)base > descr)goto fail; break; case GC_DS_BITMAP: if ((word)p - (word)base>=WORDS_TO_BYTES(BITMAP_BITS) ||((word)p&(sizeof(word)- 1)))goto fail; if (!(((word)1<<(WORDSZ - ((ptr_t)p - (ptr_t)base)- 1)) &descr))goto fail; break; case GC_DS_PROC: break; case GC_DS_PER_OBJECT: if ((signed_word)descr>=0){ descr=*(word*)((ptr_t)base+(descr&~GC_DS_TAGS)); } else { ptr_t type_descr=*(ptr_t*)base; descr=*(word*)(type_descr - (descr - (word)(GC_DS_PER_OBJECT - GC_INDIR_PER_OBJ_BIAS))); } goto retry; } return(p); } #endif fail: (*GC_is_visible_print_proc)((ptr_t)p); return(p); } GC_API void*GC_CALL GC_pre_incr (void**p,ptrdiff_t how_much) { void*initial=*p; void*result=GC_same_obj((void*)((ptr_t)initial+how_much),initial); if (!GC_all_interior_pointers){ (void)GC_is_valid_displacement(result); } return (*p=result); } GC_API void*GC_CALL GC_post_incr (void**p,ptrdiff_t how_much) { void*initial=*p; void*result=GC_same_obj((void*)((ptr_t)initial+how_much),initial); if (!GC_all_interior_pointers){ (void)GC_is_valid_displacement(result); } *p=result; return(initial); } #ifndef GC_INLINE_H #define GC_INLINE_H #if GC_GNUC_PREREQ(3,0) #define GC_EXPECT(expr,outcome)__builtin_expect(expr,outcome) #else #define GC_EXPECT(expr,outcome)(expr) #endif #ifndef GC_ASSERT #ifdef NDEBUG #define GC_ASSERT(expr) #else #include #define GC_ASSERT(expr)assert(expr) #endif #endif #ifdef __cplusplus extern "C" { #endif #ifndef GC_PREFETCH_FOR_WRITE #if GC_GNUC_PREREQ(3,0)&&!defined(GC_NO_PREFETCH_FOR_WRITE) #define GC_PREFETCH_FOR_WRITE(x)__builtin_prefetch((x),1) #else #define GC_PREFETCH_FOR_WRITE(x)(void)0 #endif #endif #define GC_I_PTRFREE 0 #define GC_I_NORMAL 1 GC_API void GC_CALL GC_generic_malloc_many(size_t,int, void**); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_malloc_kind(size_t,int); #ifdef GC_THREADS GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_malloc_kind_global(size_t,int); #else #define GC_malloc_kind_global GC_malloc_kind #endif #if defined(GC_THREADS)&&defined(AO_HAVE_store) #define GC_FAST_M_AO_STORE(my_fl,next)AO_store((volatile AO_t*)(my_fl),(AO_t)(next)) #else #define GC_FAST_M_AO_STORE(my_fl,next)(void)(*(my_fl)=(next)) #endif #define GC_FAST_MALLOC_GRANS(result,granules,tiny_fl,num_direct,kind,default_expr,init)do { if (GC_EXPECT((granules)>=GC_TINY_FREELISTS,0)){ result=(default_expr);} else { void**my_fl=(tiny_fl)+(granules);void*my_entry=*my_fl;void*next;for (;;){ if (GC_EXPECT((GC_word)my_entry > (num_direct)+GC_TINY_FREELISTS+1,1)){ next=*(void**)(my_entry);result=(void*)my_entry;GC_FAST_M_AO_STORE(my_fl,next);init;GC_PREFETCH_FOR_WRITE(next);if ((kind)!=GC_I_PTRFREE){ GC_end_stubborn_change(my_fl);GC_reachable_here(next);} GC_ASSERT(GC_size(result)>=(granules)*GC_GRANULE_BYTES);GC_ASSERT((kind)==GC_I_PTRFREE||((GC_word*)result)[1]==0);break;} if ((GC_signed_word)my_entry - (GC_signed_word)(num_direct)<=0&&my_entry!=0){ GC_FAST_M_AO_STORE(my_fl,(char*)my_entry+(granules)+1);result=(default_expr);break;} else { GC_generic_malloc_many(((granules)==0?GC_GRANULE_BYTES:GC_RAW_BYTES_FROM_INDEX(granules)),kind,my_fl);my_entry=*my_fl;if (my_entry==0){ result=(*GC_get_oom_fn())((granules)*GC_GRANULE_BYTES);break;} } } } } while (0) #define GC_WORDS_TO_WHOLE_GRANULES(n)GC_WORDS_TO_GRANULES((n)+GC_GRANULE_WORDS - 1) #define GC_MALLOC_WORDS_KIND(result,n,tiny_fl,kind,init)do { size_t granules=GC_WORDS_TO_WHOLE_GRANULES(n);GC_FAST_MALLOC_GRANS(result,granules,tiny_fl,0,kind,GC_malloc_kind(granules*GC_GRANULE_BYTES,kind),init);} while (0) #define GC_MALLOC_WORDS(result,n,tiny_fl)GC_MALLOC_WORDS_KIND(result,n,tiny_fl,GC_I_NORMAL,*(void**)(result)=0) #define GC_MALLOC_ATOMIC_WORDS(result,n,tiny_fl)GC_MALLOC_WORDS_KIND(result,n,tiny_fl,GC_I_PTRFREE,(void)0) #define GC_CONS(result,first,second,tiny_fl)do { void*l=(void*)(first);void*r=(void*)(second);GC_MALLOC_WORDS_KIND(result,2,tiny_fl,GC_I_NORMAL,(void)0);if ((result)!=0){*(void**)(result)=l;GC_PTR_STORE_AND_DIRTY((void**)(result)+1,r);GC_reachable_here(l);} } while (0) GC_API void GC_CALL GC_print_free_list(int, size_t); #ifdef __cplusplus } #endif #endif #include #ifdef GC_USE_ENTIRE_HEAP int GC_use_entire_heap=TRUE; #else int GC_use_entire_heap=FALSE; #endif #define MAX_BLACK_LIST_ALLOC (2*HBLKSIZE) #define UNIQUE_THRESHOLD 32 #define HUGE_THRESHOLD 256 #define FL_COMPRESSION 8 #define N_HBLK_FLS ((HUGE_THRESHOLD - UNIQUE_THRESHOLD)/FL_COMPRESSION+UNIQUE_THRESHOLD) #ifndef GC_GCJ_SUPPORT STATIC #endif struct hblk*GC_hblkfreelist[N_HBLK_FLS+1]={ 0 }; #ifndef GC_GCJ_SUPPORT STATIC #endif word GC_free_bytes[N_HBLK_FLS+1]={ 0 }; GC_INLINE int GC_enough_large_bytes_left(void) { int n; word bytes=GC_large_allocd_bytes; GC_ASSERT(GC_max_large_allocd_bytes<=GC_heapsize); for (n=N_HBLK_FLS;n>=0;--n){ bytes+=GC_free_bytes[n]; if (bytes>=GC_max_large_allocd_bytes)return n; } return 0; } STATIC int GC_hblk_fl_from_blocks(word blocks_needed) { if (blocks_needed<=UNIQUE_THRESHOLD)return (int)blocks_needed; if (blocks_needed>=HUGE_THRESHOLD)return N_HBLK_FLS; return (int)(blocks_needed - UNIQUE_THRESHOLD)/FL_COMPRESSION +UNIQUE_THRESHOLD; } #define PHDR(hhdr)HDR((hhdr)->hb_prev) #define NHDR(hhdr)HDR((hhdr)->hb_next) #ifdef USE_MUNMAP #define IS_MAPPED(hhdr)(((hhdr)->hb_flags&WAS_UNMAPPED)==0) #else #define IS_MAPPED(hhdr)TRUE #endif #if!defined(NO_DEBUGGING)||defined(GC_ASSERTIONS) GC_INNER word GC_compute_large_free_bytes(void) { word total_free=0; unsigned i; for (i=0;i<=N_HBLK_FLS;++i){ struct hblk*h; hdr*hhdr; for (h=GC_hblkfreelist[i];h!=0;h=hhdr->hb_next){ hhdr=HDR(h); total_free+=hhdr->hb_sz; } } return total_free; } #endif #if!defined(NO_DEBUGGING) void GC_print_hblkfreelist(void) { unsigned i; word total; for (i=0;i<=N_HBLK_FLS;++i){ struct hblk*h=GC_hblkfreelist[i]; if (0!=h)GC_printf("Free list %u (total size %lu):\n", i,(unsigned long)GC_free_bytes[i]); while (h){ hdr*hhdr=HDR(h); GC_printf("\t%p size %lu %s black listed\n", (void*)h,(unsigned long)hhdr->hb_sz, GC_is_black_listed(h,HBLKSIZE)!=0?"start": GC_is_black_listed(h,hhdr->hb_sz)!=0?"partially": "not"); h=hhdr->hb_next; } } GC_printf("GC_large_free_bytes:%lu\n", (unsigned long)GC_large_free_bytes); if ((total=GC_compute_large_free_bytes())!=GC_large_free_bytes) GC_err_printf("GC_large_free_bytes INCONSISTENT!!Should be:%lu\n", (unsigned long)total); } static int free_list_index_of(hdr*wanted) { int i; for (i=0;i<=N_HBLK_FLS;++i){ struct hblk*h; hdr*hhdr; for (h=GC_hblkfreelist[i];h!=0;h=hhdr->hb_next){ hhdr=HDR(h); if (hhdr==wanted)return i; } } return -1; } GC_API void GC_CALL GC_dump_regions(void) { unsigned i; for (i=0;i < GC_n_heap_sects;++i){ ptr_t start=GC_heap_sects[i].hs_start; size_t bytes=GC_heap_sects[i].hs_bytes; ptr_t end=start+bytes; ptr_t p; while (i+1 < GC_n_heap_sects&&GC_heap_sects[i+1].hs_start==end){ ++i; end=GC_heap_sects[i].hs_start+GC_heap_sects[i].hs_bytes; } GC_printf("***Section from %p to %p\n",(void*)start,(void*)end); for (p=start;(word)p < (word)end;){ hdr*hhdr=HDR(p); if (IS_FORWARDING_ADDR_OR_NIL(hhdr)){ GC_printf("\t%p Missing header!!(%p)\n", (void*)p,(void*)hhdr); p+=HBLKSIZE; continue; } if (HBLK_IS_FREE(hhdr)){ int correct_index=GC_hblk_fl_from_blocks( divHBLKSZ(hhdr->hb_sz)); int actual_index; GC_printf("\t%p\tfree block of size 0x%lx bytes%s\n", (void*)p,(unsigned long)(hhdr->hb_sz), IS_MAPPED(hhdr)?"":" (unmapped)"); actual_index=free_list_index_of(hhdr); if (-1==actual_index){ GC_printf("\t\tBlock not on free list %d!!\n", correct_index); } else if (correct_index!=actual_index){ GC_printf("\t\tBlock on list %d,should be on %d!!\n", actual_index,correct_index); } p+=hhdr->hb_sz; } else { GC_printf("\t%p\tused for blocks of size 0x%lx bytes\n", (void*)p,(unsigned long)(hhdr->hb_sz)); p+=HBLKSIZE*OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); } } } } #endif static GC_bool setup_header(hdr*hhdr,struct hblk*block,size_t byte_sz, int kind,unsigned flags) { word descr; #ifdef MARK_BIT_PER_GRANULE if (byte_sz > MAXOBJBYTES) flags|=LARGE_BLOCK; #endif #ifdef ENABLE_DISCLAIM if (GC_obj_kinds[kind].ok_disclaim_proc) flags|=HAS_DISCLAIM; if (GC_obj_kinds[kind].ok_mark_unconditionally) flags|=MARK_UNCONDITIONALLY; #endif hhdr->hb_sz=byte_sz; hhdr->hb_obj_kind=(unsigned char)kind; hhdr->hb_flags=(unsigned char)flags; hhdr->hb_block=block; descr=GC_obj_kinds[kind].ok_descriptor; if (GC_obj_kinds[kind].ok_relocate_descr)descr+=byte_sz; hhdr->hb_descr=descr; #ifdef MARK_BIT_PER_OBJ if (byte_sz > MAXOBJBYTES){ hhdr->hb_inv_sz=LARGE_INV_SZ; } else { word inv_sz; #if CPP_WORDSZ==64 inv_sz=((word)1<<32)/byte_sz; if (((inv_sz*byte_sz)>>32)==0)++inv_sz; #else GC_ASSERT(byte_sz>=4); inv_sz=((unsigned)1<<31)/byte_sz; inv_sz*=2; while (inv_sz*byte_sz > byte_sz)++inv_sz; #endif #ifdef INV_SZ_COMPUTATION_CHECK GC_ASSERT(((1ULL<<32)+byte_sz - 1)/byte_sz==inv_sz); #endif hhdr->hb_inv_sz=inv_sz; } #endif #ifdef MARK_BIT_PER_GRANULE { size_t granules=BYTES_TO_GRANULES(byte_sz); if (EXPECT(!GC_add_map_entry(granules),FALSE)){ hhdr->hb_sz=HBLKSIZE; hhdr->hb_descr=0; hhdr->hb_flags|=LARGE_BLOCK; hhdr->hb_map=0; return FALSE; } hhdr->hb_map=GC_obj_map[(hhdr->hb_flags&LARGE_BLOCK)!=0? 0:granules]; } #endif GC_clear_hdr_marks(hhdr); hhdr->hb_last_reclaimed=(unsigned short)GC_gc_no; return(TRUE); } STATIC void GC_remove_from_fl_at(hdr*hhdr,int index) { GC_ASSERT(((hhdr->hb_sz)&(HBLKSIZE-1))==0); if (hhdr->hb_prev==0){ GC_ASSERT(HDR(GC_hblkfreelist[index])==hhdr); GC_hblkfreelist[index]=hhdr->hb_next; } else { hdr*phdr; GET_HDR(hhdr->hb_prev,phdr); phdr->hb_next=hhdr->hb_next; } GC_ASSERT(GC_free_bytes[index]>=hhdr->hb_sz); GC_free_bytes[index]-=hhdr->hb_sz; if (0!=hhdr->hb_next){ hdr*nhdr; GC_ASSERT(!IS_FORWARDING_ADDR_OR_NIL(NHDR(hhdr))); GET_HDR(hhdr->hb_next,nhdr); nhdr->hb_prev=hhdr->hb_prev; } } GC_INLINE void GC_remove_from_fl(hdr*hhdr) { GC_remove_from_fl_at(hhdr,GC_hblk_fl_from_blocks(divHBLKSZ(hhdr->hb_sz))); } static struct hblk*get_block_ending_at(struct hblk*h) { struct hblk*p=h - 1; hdr*phdr; GET_HDR(p,phdr); while (0!=phdr&&IS_FORWARDING_ADDR_OR_NIL(phdr)){ p=FORWARDED_ADDR(p,phdr); phdr=HDR(p); } if (0!=phdr){ return p; } p=GC_prev_block(h - 1); if (p){ phdr=HDR(p); if ((ptr_t)p+phdr->hb_sz==(ptr_t)h){ return p; } } return NULL; } STATIC struct hblk*GC_free_block_ending_at(struct hblk*h) { struct hblk*p=get_block_ending_at(h); if (p){ hdr*phdr=HDR(p); if (HBLK_IS_FREE(phdr)){ return p; } } return 0; } STATIC void GC_add_to_fl(struct hblk*h,hdr*hhdr) { int index=GC_hblk_fl_from_blocks(divHBLKSZ(hhdr->hb_sz)); struct hblk*second=GC_hblkfreelist[index]; #if defined(GC_ASSERTIONS)&&!defined(USE_MUNMAP) struct hblk*next=(struct hblk*)((word)h+hhdr->hb_sz); hdr*nexthdr=HDR(next); struct hblk*prev=GC_free_block_ending_at(h); hdr*prevhdr=HDR(prev); GC_ASSERT(nexthdr==0||!HBLK_IS_FREE(nexthdr) ||(GC_heapsize&SIGNB)!=0); GC_ASSERT(prev==0||!HBLK_IS_FREE(prevhdr) ||(GC_heapsize&SIGNB)!=0); #endif GC_ASSERT(((hhdr->hb_sz)&(HBLKSIZE-1))==0); GC_hblkfreelist[index]=h; GC_free_bytes[index]+=hhdr->hb_sz; GC_ASSERT(GC_free_bytes[index]<=GC_large_free_bytes); hhdr->hb_next=second; hhdr->hb_prev=0; if (second){ hdr*second_hdr; GET_HDR(second,second_hdr); second_hdr->hb_prev=h; } hhdr->hb_flags|=FREE_BLK; } #ifdef USE_MUNMAP #ifndef MUNMAP_THRESHOLD #define MUNMAP_THRESHOLD 6 #endif GC_INNER int GC_unmap_threshold=MUNMAP_THRESHOLD; #ifdef COUNT_UNMAPPED_REGIONS static int calc_num_unmapped_regions_delta(struct hblk*h,hdr*hhdr) { struct hblk*prev=get_block_ending_at(h); struct hblk*next; GC_bool prev_unmapped=FALSE; GC_bool next_unmapped=FALSE; next=GC_next_block((struct hblk*)((ptr_t)h+hhdr->hb_sz),TRUE); if ((ptr_t)next!=GC_unmap_end((ptr_t)h,(size_t)hhdr->hb_sz)){ next=NULL; } if (prev!=NULL){ hdr*prevhdr=HDR(prev); prev_unmapped=!IS_MAPPED(prevhdr); } if (next!=NULL){ hdr*nexthdr=HDR(next); next_unmapped=!IS_MAPPED(nexthdr); } if (prev_unmapped&&next_unmapped){ return IS_MAPPED(hhdr)?-1:1; } if (!prev_unmapped&&!next_unmapped){ return IS_MAPPED(hhdr)?1:-1; } return 0; } #endif GC_INLINE void GC_adjust_num_unmapped(struct hblk*h GC_ATTR_UNUSED, hdr*hhdr GC_ATTR_UNUSED) { #ifdef COUNT_UNMAPPED_REGIONS GC_num_unmapped_regions+=calc_num_unmapped_regions_delta(h,hhdr); #endif } GC_INNER void GC_unmap_old(void) { int i; if (GC_unmap_threshold==0) return; #ifdef COUNT_UNMAPPED_REGIONS if (GC_num_unmapped_regions>=GC_UNMAPPED_REGIONS_SOFT_LIMIT) return; #endif for (i=0;i<=N_HBLK_FLS;++i){ struct hblk*h; hdr*hhdr; for (h=GC_hblkfreelist[i];0!=h;h=hhdr->hb_next){ hhdr=HDR(h); if (!IS_MAPPED(hhdr))continue; if ((unsigned short)(GC_gc_no - hhdr->hb_last_reclaimed)> (unsigned short)GC_unmap_threshold){ #ifdef COUNT_UNMAPPED_REGIONS int delta=calc_num_unmapped_regions_delta(h,hhdr); signed_word regions=GC_num_unmapped_regions+delta; if (delta>=0&®ions>=GC_UNMAPPED_REGIONS_SOFT_LIMIT){ GC_COND_LOG_PRINTF("Unmapped regions limit reached!\n"); return; } GC_num_unmapped_regions=regions; #endif GC_unmap((ptr_t)h,(size_t)hhdr->hb_sz); hhdr->hb_flags|=WAS_UNMAPPED; } } } } GC_INNER void GC_merge_unmapped(void) { int i; for (i=0;i<=N_HBLK_FLS;++i){ struct hblk*h=GC_hblkfreelist[i]; while (h!=0){ struct hblk*next; hdr*hhdr,*nexthdr; word size,nextsize; GET_HDR(h,hhdr); size=hhdr->hb_sz; next=(struct hblk*)((word)h+size); GET_HDR(next,nexthdr); if (0!=nexthdr&&HBLK_IS_FREE(nexthdr) &&(signed_word)(size+(nextsize=nexthdr->hb_sz))> 0 ){ if (IS_MAPPED(hhdr)&&!IS_MAPPED(nexthdr)){ if (size > nextsize){ GC_adjust_num_unmapped(next,nexthdr); GC_remap((ptr_t)next,nextsize); } else { GC_adjust_num_unmapped(h,hhdr); GC_unmap((ptr_t)h,size); GC_unmap_gap((ptr_t)h,size,(ptr_t)next,nextsize); hhdr->hb_flags|=WAS_UNMAPPED; } } else if (IS_MAPPED(nexthdr)&&!IS_MAPPED(hhdr)){ if (size > nextsize){ GC_adjust_num_unmapped(next,nexthdr); GC_unmap((ptr_t)next,nextsize); GC_unmap_gap((ptr_t)h,size,(ptr_t)next,nextsize); } else { GC_adjust_num_unmapped(h,hhdr); GC_remap((ptr_t)h,size); hhdr->hb_flags&=~WAS_UNMAPPED; hhdr->hb_last_reclaimed=nexthdr->hb_last_reclaimed; } } else if (!IS_MAPPED(hhdr)&&!IS_MAPPED(nexthdr)){ GC_unmap_gap((ptr_t)h,size,(ptr_t)next,nextsize); } GC_remove_from_fl_at(hhdr,i); GC_remove_from_fl(nexthdr); hhdr->hb_sz+=nexthdr->hb_sz; GC_remove_header(next); GC_add_to_fl(h,hhdr); h=GC_hblkfreelist[i]; } else { h=hhdr->hb_next; } } } } #endif STATIC struct hblk*GC_get_first_part(struct hblk*h,hdr*hhdr, size_t bytes,int index) { word total_size=hhdr->hb_sz; struct hblk*rest; hdr*rest_hdr; GC_ASSERT((total_size&(HBLKSIZE-1))==0); GC_remove_from_fl_at(hhdr,index); if (total_size==bytes)return h; rest=(struct hblk*)((word)h+bytes); rest_hdr=GC_install_header(rest); if (0==rest_hdr){ WARN("Header allocation failed:dropping block\n",0); return(0); } rest_hdr->hb_sz=total_size - bytes; rest_hdr->hb_flags=0; #ifdef GC_ASSERTIONS hhdr->hb_flags&=~FREE_BLK; #endif GC_add_to_fl(rest,rest_hdr); return h; } STATIC void GC_split_block(struct hblk*h,hdr*hhdr,struct hblk*n, hdr*nhdr,int index) { word total_size=hhdr->hb_sz; word h_size=(word)n - (word)h; struct hblk*prev=hhdr->hb_prev; struct hblk*next=hhdr->hb_next; nhdr->hb_prev=prev; nhdr->hb_next=next; nhdr->hb_sz=total_size - h_size; nhdr->hb_flags=0; if (prev){ HDR(prev)->hb_next=n; } else { GC_hblkfreelist[index]=n; } if (next){ HDR(next)->hb_prev=n; } GC_ASSERT(GC_free_bytes[index] > h_size); GC_free_bytes[index]-=h_size; #ifdef USE_MUNMAP hhdr->hb_last_reclaimed=(unsigned short)GC_gc_no; #endif hhdr->hb_sz=h_size; GC_add_to_fl(h,hhdr); nhdr->hb_flags|=FREE_BLK; } STATIC struct hblk* GC_allochblk_nth(size_t sz,int kind,unsigned flags,int n, int may_split); #define AVOID_SPLIT_REMAPPED 2 GC_INNER struct hblk* GC_allochblk(size_t sz,int kind,unsigned flags) { word blocks; int start_list; struct hblk*result; int may_split; int split_limit; GC_ASSERT((sz&(GRANULE_BYTES - 1))==0); blocks=OBJ_SZ_TO_BLOCKS_CHECKED(sz); if ((signed_word)(blocks*HBLKSIZE)< 0){ return 0; } start_list=GC_hblk_fl_from_blocks(blocks); result=GC_allochblk_nth(sz,kind,flags,start_list,FALSE); if (0!=result)return result; may_split=TRUE; if (GC_use_entire_heap||GC_dont_gc ||USED_HEAP_SIZE < GC_requested_heapsize ||GC_incremental||!GC_should_collect()){ split_limit=N_HBLK_FLS; } else if (GC_finalizer_bytes_freed > (GC_heapsize>>4)){ split_limit=0; } else { split_limit=GC_enough_large_bytes_left(); #ifdef USE_MUNMAP if (split_limit > 0) may_split=AVOID_SPLIT_REMAPPED; #endif } if (start_list < UNIQUE_THRESHOLD){ ++start_list; } for (;start_list<=split_limit;++start_list){ result=GC_allochblk_nth(sz,kind,flags,start_list,may_split); if (0!=result) break; } return result; } STATIC long GC_large_alloc_warn_suppressed=0; STATIC struct hblk* GC_allochblk_nth(size_t sz,int kind,unsigned flags,int n,int may_split) { struct hblk*hbp; hdr*hhdr; struct hblk*thishbp; hdr*thishdr; signed_word size_needed=HBLKSIZE*OBJ_SZ_TO_BLOCKS_CHECKED(sz); for (hbp=GC_hblkfreelist[n];;hbp=hhdr->hb_next){ signed_word size_avail; if (hbp){ } else { return NULL; } GET_HDR(hbp,hhdr); size_avail=(signed_word)hhdr->hb_sz; if (size_avail < size_needed)continue; if (size_avail!=size_needed){ if (!may_split)continue; thishbp=hhdr->hb_next; if (thishbp){ signed_word next_size; GET_HDR(thishbp,thishdr); next_size=(signed_word)(thishdr->hb_sz); if (next_size < size_avail &&next_size>=size_needed &&!GC_is_black_listed(thishbp,(word)size_needed)){ continue; } } } if (!IS_UNCOLLECTABLE(kind)&&(kind!=PTRFREE ||size_needed > (signed_word)MAX_BLACK_LIST_ALLOC)){ struct hblk*lasthbp=hbp; ptr_t search_end=(ptr_t)hbp+size_avail - size_needed; signed_word orig_avail=size_avail; signed_word eff_size_needed=(flags&IGNORE_OFF_PAGE)!=0? (signed_word)HBLKSIZE :size_needed; while ((word)lasthbp<=(word)search_end &&(thishbp=GC_is_black_listed(lasthbp, (word)eff_size_needed))!=0){ lasthbp=thishbp; } size_avail-=(ptr_t)lasthbp - (ptr_t)hbp; thishbp=lasthbp; if (size_avail>=size_needed){ if (thishbp!=hbp){ #ifdef USE_MUNMAP if (may_split==AVOID_SPLIT_REMAPPED&&!IS_MAPPED(hhdr)) continue; #endif thishdr=GC_install_header(thishbp); if (0!=thishdr){ #ifdef USE_MUNMAP if (!IS_MAPPED(hhdr)){ GC_adjust_num_unmapped(hbp,hhdr); GC_remap((ptr_t)hbp,(size_t)hhdr->hb_sz); hhdr->hb_flags&=~WAS_UNMAPPED; } #endif GC_split_block(hbp,hhdr,thishbp,thishdr,n); hbp=thishbp; hhdr=thishdr; } } } else if (size_needed > (signed_word)BL_LIMIT &&orig_avail - size_needed > (signed_word)BL_LIMIT){ if (++GC_large_alloc_warn_suppressed >=GC_large_alloc_warn_interval){ WARN("Repeated allocation of very large block " "(appr. size %" WARN_PRIdPTR "):\n" "\tMay lead to memory leak and poor performance\n", size_needed); GC_large_alloc_warn_suppressed=0; } size_avail=orig_avail; } else if (size_avail==0 &&size_needed==(signed_word)HBLKSIZE &&IS_MAPPED(hhdr)){ if (!GC_find_leak){ static unsigned count=0; if ((++count&3)==0){ word total_size=hhdr->hb_sz; struct hblk*limit=hbp+divHBLKSZ(total_size); struct hblk*h; struct hblk*prev=hhdr->hb_prev; GC_large_free_bytes-=total_size; GC_bytes_dropped+=total_size; GC_remove_from_fl_at(hhdr,n); for (h=hbp;(word)h < (word)limit;h++){ if (h!=hbp){ hhdr=GC_install_header(h); } if (NULL!=hhdr){ (void)setup_header(hhdr,h,HBLKSIZE,PTRFREE,0); if (GC_debugging_started){ BZERO(h,HBLKSIZE); } } } hbp=prev; if (0==hbp){ return GC_allochblk_nth(sz,kind,flags,n,may_split); } hhdr=HDR(hbp); } } } } if( size_avail>=size_needed){ #ifdef USE_MUNMAP if (!IS_MAPPED(hhdr)){ GC_adjust_num_unmapped(hbp,hhdr); GC_remap((ptr_t)hbp,(size_t)hhdr->hb_sz); hhdr->hb_flags&=~WAS_UNMAPPED; } #endif hbp=GC_get_first_part(hbp,hhdr,size_needed,n); break; } } if (0==hbp)return 0; if (!GC_install_counts(hbp,(word)size_needed))return(0); if (!setup_header(hhdr,hbp,sz,kind,flags)){ GC_remove_counts(hbp,(word)size_needed); return(0); } #ifndef GC_DISABLE_INCREMENTAL GC_ASSERT((size_needed&(HBLKSIZE-1))==0); GC_remove_protection(hbp,divHBLKSZ(size_needed), (hhdr->hb_descr==0)); #endif GC_fail_count=0; GC_large_free_bytes-=size_needed; GC_ASSERT(IS_MAPPED(hhdr)); return( hbp); } GC_INNER void GC_freehblk(struct hblk*hbp) { struct hblk*next,*prev; hdr*hhdr,*prevhdr,*nexthdr; word size; GET_HDR(hbp,hhdr); size=HBLKSIZE*OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); if ((size&SIGNB)!=0) ABORT("Deallocating excessively large block. Too large an allocation?"); GC_remove_counts(hbp,size); hhdr->hb_sz=size; #ifdef USE_MUNMAP hhdr->hb_last_reclaimed=(unsigned short)GC_gc_no; #endif if (HBLK_IS_FREE(hhdr)){ ABORT_ARG1("Duplicate large block deallocation", " of %p",(void*)hbp); } GC_ASSERT(IS_MAPPED(hhdr)); hhdr->hb_flags|=FREE_BLK; next=(struct hblk*)((ptr_t)hbp+size); GET_HDR(next,nexthdr); prev=GC_free_block_ending_at(hbp); if(0!=nexthdr&&HBLK_IS_FREE(nexthdr)&&IS_MAPPED(nexthdr) &&(signed_word)(hhdr->hb_sz+nexthdr->hb_sz)> 0 ){ GC_remove_from_fl(nexthdr); hhdr->hb_sz+=nexthdr->hb_sz; GC_remove_header(next); } if (prev){ prevhdr=HDR(prev); if (IS_MAPPED(prevhdr) &&(signed_word)(hhdr->hb_sz+prevhdr->hb_sz)> 0){ GC_remove_from_fl(prevhdr); prevhdr->hb_sz+=hhdr->hb_sz; #ifdef USE_MUNMAP prevhdr->hb_last_reclaimed=(unsigned short)GC_gc_no; #endif GC_remove_header(hbp); hbp=prev; hhdr=prevhdr; } } GC_large_free_bytes+=size; GC_add_to_fl(hbp,hhdr); } #include #if!defined(MACOS)&&!defined(MSWINCE) #include #if!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PSP2)&&!defined(__CC_ARM) #include #endif #endif word GC_non_gc_bytes=0; word GC_gc_no=0; #ifndef NO_CLOCK static unsigned long full_gc_total_time=0; static unsigned full_gc_total_ns_frac=0; static GC_bool measure_performance=FALSE; GC_API void GC_CALL GC_start_performance_measurement(void) { measure_performance=TRUE; } GC_API unsigned long GC_CALL GC_get_full_gc_total_time(void) { return full_gc_total_time; } #endif #ifndef GC_DISABLE_INCREMENTAL GC_INNER GC_bool GC_incremental=FALSE; #endif GC_API int GC_CALL GC_is_incremental_mode(void) { return (int)GC_incremental; } #ifdef THREADS int GC_parallel=FALSE; #endif #if defined(GC_FULL_FREQ)&&!defined(CPPCHECK) int GC_full_freq=GC_FULL_FREQ; #else int GC_full_freq=19; #endif STATIC GC_bool GC_need_full_gc=FALSE; #ifdef THREAD_LOCAL_ALLOC GC_INNER GC_bool GC_world_stopped=FALSE; #endif STATIC word GC_used_heap_size_after_full=0; EXTERN_C_BEGIN extern const char*const GC_copyright[]; EXTERN_C_END const char*const GC_copyright[]= {"Copyright 1988,1989 Hans-J. Boehm and Alan J. Demers ", "Copyright (c)1991-1995 by Xerox Corporation. All rights reserved. ", "Copyright (c)1996-1998 by Silicon Graphics. All rights reserved. ", "Copyright (c)1999-2009 by Hewlett-Packard Company. All rights reserved. ", "Copyright (c)2008-2020 Ivan Maidanski ", "THIS MATERIAL IS PROVIDED AS IS,WITH ABSOLUTELY NO WARRANTY", " EXPRESSED OR IMPLIED. ANY USE IS AT YOUR OWN RISK.", "See source code for details." }; #ifndef GC_NO_VERSION_VAR EXTERN_C_BEGIN extern const unsigned GC_version; EXTERN_C_END const unsigned GC_version=((GC_VERSION_MAJOR<<16)| (GC_VERSION_MINOR<<8)|GC_VERSION_MICRO); #endif GC_API unsigned GC_CALL GC_get_version(void) { return (GC_VERSION_MAJOR<<16)|(GC_VERSION_MINOR<<8)| GC_VERSION_MICRO; } #ifdef GC_DONT_EXPAND int GC_dont_expand=TRUE; #else int GC_dont_expand=FALSE; #endif #if defined(GC_FREE_SPACE_DIVISOR)&&!defined(CPPCHECK) word GC_free_space_divisor=GC_FREE_SPACE_DIVISOR; #else word GC_free_space_divisor=3; #endif GC_INNER int GC_CALLBACK GC_never_stop_func(void) { return(0); } #if defined(GC_TIME_LIMIT)&&!defined(CPPCHECK) unsigned long GC_time_limit=GC_TIME_LIMIT; #elif defined(PARALLEL_MARK) unsigned long GC_time_limit=GC_TIME_UNLIMITED; #else unsigned long GC_time_limit=50; #endif #ifndef NO_CLOCK STATIC unsigned long GC_time_lim_nsec=0; #define TV_NSEC_LIMIT (1000UL*1000) GC_API void GC_CALL GC_set_time_limit_tv(struct GC_timeval_s tv) { GC_ASSERT(tv.tv_ms<=GC_TIME_UNLIMITED); GC_ASSERT(tv.tv_nsec < TV_NSEC_LIMIT); GC_time_limit=tv.tv_ms; GC_time_lim_nsec=tv.tv_nsec; } GC_API struct GC_timeval_s GC_CALL GC_get_time_limit_tv(void) { struct GC_timeval_s tv; tv.tv_ms=GC_time_limit; tv.tv_nsec=GC_time_lim_nsec; return tv; } STATIC CLOCK_TYPE GC_start_time=CLOCK_TYPE_INITIALIZER; #endif STATIC int GC_n_attempts=0; STATIC GC_stop_func GC_default_stop_func=GC_never_stop_func; GC_API void GC_CALL GC_set_stop_func(GC_stop_func stop_func) { DCL_LOCK_STATE; GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func)); LOCK(); GC_default_stop_func=stop_func; UNLOCK(); } GC_API GC_stop_func GC_CALL GC_get_stop_func(void) { GC_stop_func stop_func; DCL_LOCK_STATE; LOCK(); stop_func=GC_default_stop_func; UNLOCK(); return stop_func; } #if defined(GC_DISABLE_INCREMENTAL)||defined(NO_CLOCK) #define GC_timeout_stop_func GC_default_stop_func #else STATIC int GC_CALLBACK GC_timeout_stop_func (void) { CLOCK_TYPE current_time; static unsigned count=0; unsigned long time_diff,nsec_diff; if ((*GC_default_stop_func)()) return(1); if ((count++&3)!=0)return(0); GET_TIME(current_time); time_diff=MS_TIME_DIFF(current_time,GC_start_time); nsec_diff=NS_FRAC_TIME_DIFF(current_time,GC_start_time); #if defined(CPPCHECK) GC_noop1((word)&nsec_diff); #endif if (time_diff>=GC_time_limit &&(time_diff > GC_time_limit||nsec_diff>=GC_time_lim_nsec)){ GC_COND_LOG_PRINTF("Abandoning stopped marking after %lu ms %lu ns" " (attempt %d)\n", time_diff,nsec_diff,GC_n_attempts); return 1; } return(0); } #endif #ifdef THREADS GC_INNER word GC_total_stacksize=0; #endif static size_t min_bytes_allocd_minimum=1; GC_API void GC_CALL GC_set_min_bytes_allocd(size_t value) { GC_ASSERT(value > 0); min_bytes_allocd_minimum=value; } GC_API size_t GC_CALL GC_get_min_bytes_allocd(void) { return min_bytes_allocd_minimum; } static word min_bytes_allocd(void) { word result; word stack_size; word total_root_size; word scan_size; #ifdef THREADS if (GC_need_to_lock){ stack_size=GC_total_stacksize; #ifdef DEBUG_THREADS GC_log_printf("Total stacks size:%lu\n", (unsigned long)stack_size); #endif } else #endif { #ifdef STACK_NOT_SCANNED stack_size=0; #elif defined(STACK_GROWS_UP) stack_size=GC_approx_sp()- GC_stackbottom; #else stack_size=GC_stackbottom - GC_approx_sp(); #endif } total_root_size=2*stack_size+GC_root_size; scan_size=2*GC_composite_in_use+GC_atomic_in_use/4 +total_root_size; result=scan_size/GC_free_space_divisor; if (GC_incremental){ result/=2; } return result > min_bytes_allocd_minimum ?result:min_bytes_allocd_minimum; } STATIC word GC_non_gc_bytes_at_gc=0; STATIC word GC_adj_bytes_allocd(void) { signed_word result; signed_word expl_managed=(signed_word)GC_non_gc_bytes - (signed_word)GC_non_gc_bytes_at_gc; result=(signed_word)GC_bytes_allocd +(signed_word)GC_bytes_dropped - (signed_word)GC_bytes_freed +(signed_word)GC_finalizer_bytes_freed - expl_managed; if (result > (signed_word)GC_bytes_allocd){ result=GC_bytes_allocd; } result+=GC_bytes_finalized; if (result < (signed_word)(GC_bytes_allocd>>3)){ return(GC_bytes_allocd>>3); } else { return(result); } } STATIC void GC_clear_a_few_frames(void) { #ifndef CLEAR_NWORDS #define CLEAR_NWORDS 64 #endif volatile word frames[CLEAR_NWORDS]; BZERO((word*)frames,CLEAR_NWORDS*sizeof(word)); } STATIC word GC_collect_at_heapsize=GC_WORD_MAX; GC_INNER GC_bool GC_should_collect(void) { static word last_min_bytes_allocd; static word last_gc_no; if (last_gc_no!=GC_gc_no){ last_min_bytes_allocd=min_bytes_allocd(); last_gc_no=GC_gc_no; } return(GC_adj_bytes_allocd()>=last_min_bytes_allocd ||GC_heapsize>=GC_collect_at_heapsize); } GC_start_callback_proc GC_start_call_back=0; GC_API void GC_CALL GC_set_start_callback(GC_start_callback_proc fn) { DCL_LOCK_STATE; LOCK(); GC_start_call_back=fn; UNLOCK(); } GC_API GC_start_callback_proc GC_CALL GC_get_start_callback(void) { GC_start_callback_proc fn; DCL_LOCK_STATE; LOCK(); fn=GC_start_call_back; UNLOCK(); return fn; } GC_INLINE void GC_notify_full_gc(void) { if (GC_start_call_back!=0){ (*GC_start_call_back)(); } } STATIC GC_bool GC_is_full_gc=FALSE; STATIC GC_bool GC_stopped_mark(GC_stop_func stop_func); STATIC void GC_finish_collection(void); STATIC void GC_maybe_gc(void) { GC_ASSERT(I_HOLD_LOCK()); ASSERT_CANCEL_DISABLED(); if (GC_should_collect()){ static int n_partial_gcs=0; if (!GC_incremental){ GC_try_to_collect_inner(GC_never_stop_func); n_partial_gcs=0; return; } else { #ifdef PARALLEL_MARK if (GC_parallel) GC_wait_for_reclaim(); #endif if (GC_need_full_gc||n_partial_gcs>=GC_full_freq){ GC_COND_LOG_PRINTF( "***>Full mark for collection #%lu after %lu allocd bytes\n", (unsigned long)GC_gc_no+1,(unsigned long)GC_bytes_allocd); GC_promote_black_lists(); (void)GC_reclaim_all((GC_stop_func)0,TRUE); GC_notify_full_gc(); GC_clear_marks(); n_partial_gcs=0; GC_is_full_gc=TRUE; } else { n_partial_gcs++; } } #ifndef NO_CLOCK if (GC_time_limit!=GC_TIME_UNLIMITED){ GET_TIME(GC_start_time);} #endif if (GC_stopped_mark(GC_time_limit==GC_TIME_UNLIMITED? GC_never_stop_func:GC_timeout_stop_func)){ #ifdef SAVE_CALL_CHAIN GC_save_callers(GC_last_stack); #endif GC_finish_collection(); } else { if (!GC_is_full_gc){ GC_n_attempts++; } } } } STATIC GC_on_collection_event_proc GC_on_collection_event=0; GC_API void GC_CALL GC_set_on_collection_event(GC_on_collection_event_proc fn) { DCL_LOCK_STATE; LOCK(); GC_on_collection_event=fn; UNLOCK(); } GC_API GC_on_collection_event_proc GC_CALL GC_get_on_collection_event(void) { GC_on_collection_event_proc fn; DCL_LOCK_STATE; LOCK(); fn=GC_on_collection_event; UNLOCK(); return fn; } GC_INNER GC_bool GC_try_to_collect_inner(GC_stop_func stop_func) { #ifndef NO_CLOCK CLOCK_TYPE start_time=CLOCK_TYPE_INITIALIZER; GC_bool start_time_valid; #endif ASSERT_CANCEL_DISABLED(); GC_ASSERT(I_HOLD_LOCK()); if (GC_dont_gc||(*stop_func)())return FALSE; if (GC_on_collection_event) GC_on_collection_event(GC_EVENT_START); if (GC_incremental&&GC_collection_in_progress()){ GC_COND_LOG_PRINTF( "GC_try_to_collect_inner:finishing collection in progress\n"); while(GC_collection_in_progress()){ if ((*stop_func)()){ return(FALSE); } ENTER_GC(); GC_collect_a_little_inner(1); EXIT_GC(); } } GC_notify_full_gc(); #ifndef NO_CLOCK start_time_valid=FALSE; if ((GC_print_stats|(int)measure_performance)!=0){ if (GC_print_stats) GC_log_printf("Initiating full world-stop collection!\n"); start_time_valid=TRUE; GET_TIME(start_time); } #endif GC_promote_black_lists(); #ifdef PARALLEL_MARK if (GC_parallel) GC_wait_for_reclaim(); #endif if ((GC_find_leak||stop_func!=GC_never_stop_func) &&!GC_reclaim_all(stop_func,FALSE)){ return(FALSE); } GC_invalidate_mark_state(); GC_clear_marks(); #ifdef SAVE_CALL_CHAIN GC_save_callers(GC_last_stack); #endif GC_is_full_gc=TRUE; if (!GC_stopped_mark(stop_func)){ if (!GC_incremental){ GC_invalidate_mark_state(); GC_unpromote_black_lists(); } return(FALSE); } GC_finish_collection(); #ifndef NO_CLOCK if (start_time_valid){ CLOCK_TYPE current_time; unsigned long time_diff,ns_frac_diff; GET_TIME(current_time); time_diff=MS_TIME_DIFF(current_time,start_time); ns_frac_diff=NS_FRAC_TIME_DIFF(current_time,start_time); if (measure_performance){ full_gc_total_time+=time_diff; full_gc_total_ns_frac+=(unsigned)ns_frac_diff; if (full_gc_total_ns_frac>=1000000U){ full_gc_total_ns_frac-=1000000U; full_gc_total_time++; } } if (GC_print_stats) GC_log_printf("Complete collection took %lu ms %lu ns\n", time_diff,ns_frac_diff); } #endif if (GC_on_collection_event) GC_on_collection_event(GC_EVENT_END); return(TRUE); } #ifndef GC_RATE #define GC_RATE 10 #endif #ifndef MAX_PRIOR_ATTEMPTS #define MAX_PRIOR_ATTEMPTS 1 #endif STATIC int GC_deficit=0; STATIC int GC_rate=GC_RATE; GC_API void GC_CALL GC_set_rate(int value) { GC_ASSERT(value > 0); GC_rate=value; } GC_API int GC_CALL GC_get_rate(void) { return GC_rate; } static int max_prior_attempts=MAX_PRIOR_ATTEMPTS; GC_API void GC_CALL GC_set_max_prior_attempts(int value) { GC_ASSERT(value>=0); max_prior_attempts=value; } GC_API int GC_CALL GC_get_max_prior_attempts(void) { return max_prior_attempts; } GC_INNER void GC_collect_a_little_inner(int n) { IF_CANCEL(int cancel_state;) GC_ASSERT(I_HOLD_LOCK()); if (GC_dont_gc)return; DISABLE_CANCEL(cancel_state); if (GC_incremental&&GC_collection_in_progress()){ int i; int max_deficit=GC_rate*n; #ifdef PARALLEL_MARK if (GC_time_limit!=GC_TIME_UNLIMITED) GC_parallel_mark_disabled=TRUE; #endif for (i=GC_deficit;i < max_deficit;i++){ if (GC_mark_some(NULL)) break; } #ifdef PARALLEL_MARK GC_parallel_mark_disabled=FALSE; #endif if (i < max_deficit){ #ifdef SAVE_CALL_CHAIN GC_save_callers(GC_last_stack); #endif #ifdef PARALLEL_MARK if (GC_parallel) GC_wait_for_reclaim(); #endif if (GC_n_attempts < max_prior_attempts &&GC_time_limit!=GC_TIME_UNLIMITED){ #ifndef NO_CLOCK GET_TIME(GC_start_time); #endif if (GC_stopped_mark(GC_timeout_stop_func)){ GC_finish_collection(); } else { GC_n_attempts++; } } else { (void)GC_stopped_mark(GC_never_stop_func); GC_finish_collection(); } } if (GC_deficit > 0){ GC_deficit-=max_deficit; if (GC_deficit < 0) GC_deficit=0; } } else { GC_maybe_gc(); } RESTORE_CANCEL(cancel_state); } GC_INNER void (*GC_check_heap)(void)=0; GC_INNER void (*GC_print_all_smashed)(void)=0; GC_API int GC_CALL GC_collect_a_little(void) { int result; DCL_LOCK_STATE; LOCK(); ENTER_GC(); GC_collect_a_little_inner(1); EXIT_GC(); result=(int)GC_collection_in_progress(); UNLOCK(); if (!result&&GC_debugging_started)GC_print_all_smashed(); return(result); } #ifndef NO_CLOCK static unsigned world_stopped_total_time=0; static unsigned world_stopped_total_divisor=0; #ifndef MAX_TOTAL_TIME_DIVISOR #define MAX_TOTAL_TIME_DIVISOR 1000 #endif #endif #ifdef USE_MUNMAP #define IF_USE_MUNMAP(x)x #define COMMA_IF_USE_MUNMAP(x),x #else #define IF_USE_MUNMAP(x) #define COMMA_IF_USE_MUNMAP(x) #endif STATIC GC_bool GC_stopped_mark(GC_stop_func stop_func) { int i; #ifndef NO_CLOCK CLOCK_TYPE start_time=CLOCK_TYPE_INITIALIZER; #endif GC_ASSERT(I_HOLD_LOCK()); #if!defined(REDIRECT_MALLOC)&&defined(USE_WINALLOC) GC_add_current_malloc_heap(); #endif #if defined(REGISTER_LIBRARIES_EARLY) GC_cond_register_dynamic_libraries(); #endif #ifndef NO_CLOCK if (GC_PRINT_STATS_FLAG) GET_TIME(start_time); #endif #if!defined(GC_NO_FINALIZATION)&&!defined(GC_TOGGLE_REFS_NOT_NEEDED) GC_process_togglerefs(); #endif #ifdef THREADS if (GC_on_collection_event) GC_on_collection_event(GC_EVENT_PRE_STOP_WORLD); #endif STOP_WORLD(); #ifdef THREADS if (GC_on_collection_event) GC_on_collection_event(GC_EVENT_POST_STOP_WORLD); #endif #ifdef THREAD_LOCAL_ALLOC GC_world_stopped=TRUE; #endif GC_COND_LOG_PRINTF( "\n--> Marking for collection #%lu after %lu allocated bytes\n", (unsigned long)GC_gc_no+1,(unsigned long)GC_bytes_allocd); #ifdef MAKE_BACK_GRAPH if (GC_print_back_height){ GC_build_back_graph(); } #endif if (GC_on_collection_event) GC_on_collection_event(GC_EVENT_MARK_START); GC_clear_a_few_frames(); GC_noop6(0,0,0,0,0,0); GC_initiate_gc(); #ifdef PARALLEL_MARK if (stop_func!=GC_never_stop_func) GC_parallel_mark_disabled=TRUE; #endif for (i=0;!(*stop_func)();i++){ if (GC_mark_some(GC_approx_sp())){ #ifdef PARALLEL_MARK if (GC_parallel&&GC_parallel_mark_disabled){ GC_COND_LOG_PRINTF("Stopped marking done after %d iterations" " with disabled parallel marker\n",i); } #endif i=-1; break; } } #ifdef PARALLEL_MARK GC_parallel_mark_disabled=FALSE; #endif if (i>=0){ GC_COND_LOG_PRINTF("Abandoned stopped marking after" " %d iterations\n",i); GC_deficit=i; #ifdef THREAD_LOCAL_ALLOC GC_world_stopped=FALSE; #endif #ifdef THREADS if (GC_on_collection_event) GC_on_collection_event(GC_EVENT_PRE_START_WORLD); #endif START_WORLD(); #ifdef THREADS if (GC_on_collection_event) GC_on_collection_event(GC_EVENT_POST_START_WORLD); #endif return FALSE; } GC_gc_no++; GC_DBGLOG_PRINTF("GC #%lu freed %ld bytes,heap %lu KiB" IF_USE_MUNMAP(" (+%lu KiB unmapped)")"\n", (unsigned long)GC_gc_no,(long)GC_bytes_found, TO_KiB_UL(GC_heapsize - GC_unmapped_bytes) COMMA_IF_USE_MUNMAP(TO_KiB_UL(GC_unmapped_bytes))); if (GC_debugging_started){ (*GC_check_heap)(); } if (GC_on_collection_event){ GC_on_collection_event(GC_EVENT_MARK_END); #ifdef THREADS GC_on_collection_event(GC_EVENT_PRE_START_WORLD); #endif } #ifdef THREAD_LOCAL_ALLOC GC_world_stopped=FALSE; #endif START_WORLD(); #ifdef THREADS if (GC_on_collection_event) GC_on_collection_event(GC_EVENT_POST_START_WORLD); #endif #ifndef NO_CLOCK if (GC_PRINT_STATS_FLAG){ unsigned long time_diff; unsigned total_time,divisor; CLOCK_TYPE current_time; GET_TIME(current_time); time_diff=MS_TIME_DIFF(current_time,start_time); total_time=world_stopped_total_time; divisor=world_stopped_total_divisor; if ((int)total_time < 0||divisor>=MAX_TOTAL_TIME_DIVISOR){ total_time>>=1; divisor>>=1; } total_time+=time_diff < (((unsigned)-1)>>1)? (unsigned)time_diff:((unsigned)-1)>>1; world_stopped_total_time=total_time; world_stopped_total_divisor=++divisor; GC_ASSERT(divisor!=0); GC_log_printf("World-stopped marking took %lu ms %lu ns" " (%u ms in average)\n", time_diff,NS_FRAC_TIME_DIFF(current_time,start_time), total_time/divisor); } #endif return(TRUE); } GC_INNER void GC_set_fl_marks(ptr_t q) { if (q){ struct hblk*h=HBLKPTR(q); struct hblk*last_h=h; hdr*hhdr=HDR(h); IF_PER_OBJ(word sz=hhdr->hb_sz;) for (;;){ word bit_no=MARK_BIT_NO((ptr_t)q - (ptr_t)h,sz); if (!mark_bit_from_hdr(hhdr,bit_no)){ set_mark_bit_from_hdr(hhdr,bit_no); ++hhdr->hb_n_marks; } q=(ptr_t)obj_link(q); if (q==NULL) break; h=HBLKPTR(q); if (h!=last_h){ last_h=h; hhdr=HDR(h); IF_PER_OBJ(sz=hhdr->hb_sz;) } } } } #if defined(GC_ASSERTIONS)&&defined(THREAD_LOCAL_ALLOC) void GC_check_fl_marks(void**pfreelist) { #if defined(AO_HAVE_load_acquire_read)&&!defined(THREAD_SANITIZER) AO_t*list=(AO_t*)AO_load_acquire_read((AO_t*)pfreelist); AO_t*prev; AO_t*p; if ((word)list<=HBLKSIZE)return; prev=(AO_t*)pfreelist; for (p=list;p!=NULL;){ AO_t*next; if (!GC_is_marked(p)){ ABORT_ARG2("Unmarked local free list entry", ":object %p on list %p",(void*)p,(void*)list); } next=(AO_t*)AO_load_acquire_read(p); if (AO_load(prev)!=(AO_t)p) break; prev=p; p=next; } #else (void)pfreelist; #endif } #endif STATIC void GC_clear_fl_marks(ptr_t q) { struct hblk*h=HBLKPTR(q); struct hblk*last_h=h; hdr*hhdr=HDR(h); word sz=hhdr->hb_sz; for (;;){ word bit_no=MARK_BIT_NO((ptr_t)q - (ptr_t)h,sz); if (mark_bit_from_hdr(hhdr,bit_no)){ size_t n_marks=hhdr->hb_n_marks; GC_ASSERT(n_marks!=0); clear_mark_bit_from_hdr(hhdr,bit_no); n_marks--; #ifdef PARALLEL_MARK if (0!=n_marks||!GC_parallel){ hhdr->hb_n_marks=n_marks; } #else hhdr->hb_n_marks=n_marks; #endif } GC_bytes_found-=sz; q=(ptr_t)obj_link(q); if (q==NULL) break; h=HBLKPTR(q); if (h!=last_h){ last_h=h; hhdr=HDR(h); sz=hhdr->hb_sz; } } } #if defined(GC_ASSERTIONS)&&defined(THREAD_LOCAL_ALLOC) void GC_check_tls(void); #endif GC_on_heap_resize_proc GC_on_heap_resize=0; GC_INLINE int GC_compute_heap_usage_percent(void) { word used=GC_composite_in_use+GC_atomic_in_use; word heap_sz=GC_heapsize - GC_unmapped_bytes; #if defined(CPPCHECK) word limit=(GC_WORD_MAX>>1)/50; #else const word limit=GC_WORD_MAX/100; #endif return used>=heap_sz?0:used < limit? (int)((used*100)/heap_sz):(int)(used/(heap_sz/100)); } STATIC void GC_finish_collection(void) { #ifndef NO_CLOCK CLOCK_TYPE start_time=CLOCK_TYPE_INITIALIZER; CLOCK_TYPE finalize_time=CLOCK_TYPE_INITIALIZER; #endif GC_ASSERT(I_HOLD_LOCK()); #if defined(GC_ASSERTIONS)&&defined(THREAD_LOCAL_ALLOC)&&!defined(DBG_HDRS_ALL) GC_check_tls(); #endif #ifndef NO_CLOCK if (GC_print_stats) GET_TIME(start_time); #endif if (GC_on_collection_event) GC_on_collection_event(GC_EVENT_RECLAIM_START); #ifndef GC_GET_HEAP_USAGE_NOT_NEEDED if (GC_bytes_found > 0) GC_reclaimed_bytes_before_gc+=(word)GC_bytes_found; #endif GC_bytes_found=0; #if defined(LINUX)&&defined(__ELF__)&&!defined(SMALL_CONFIG) if (GETENV("GC_PRINT_ADDRESS_MAP")!=0){ GC_print_address_map(); } #endif COND_DUMP; if (GC_find_leak){ word size; unsigned kind; ptr_t q; for (kind=0;kind < GC_n_kinds;kind++){ for (size=1;size<=MAXOBJGRANULES;size++){ q=(ptr_t)GC_obj_kinds[kind].ok_freelist[size]; if (q!=NULL) GC_set_fl_marks(q); } } GC_start_reclaim(TRUE); } #ifndef GC_NO_FINALIZATION GC_finalize(); #endif #ifndef NO_CLOCK if (GC_print_stats) GET_TIME(finalize_time); #endif if (GC_print_back_height){ #ifdef MAKE_BACK_GRAPH GC_traverse_back_graph(); #elif!defined(SMALL_CONFIG) GC_err_printf("Back height not available:" "Rebuild collector with -DMAKE_BACK_GRAPH\n"); #endif } { word size; ptr_t q; unsigned kind; for (kind=0;kind < GC_n_kinds;kind++){ for (size=1;size<=MAXOBJGRANULES;size++){ q=(ptr_t)GC_obj_kinds[kind].ok_freelist[size]; if (q!=NULL) GC_clear_fl_marks(q); } } } GC_VERBOSE_LOG_PRINTF("Bytes recovered before sweep - f.l. count=%ld\n", (long)GC_bytes_found); GC_start_reclaim(FALSE); GC_DBGLOG_PRINTF("In-use heap:%d%% (%lu KiB pointers+%lu KiB other)\n", GC_compute_heap_usage_percent(), TO_KiB_UL(GC_composite_in_use), TO_KiB_UL(GC_atomic_in_use)); if (GC_is_full_gc){ GC_used_heap_size_after_full=USED_HEAP_SIZE; GC_need_full_gc=FALSE; } else { GC_need_full_gc=USED_HEAP_SIZE - GC_used_heap_size_after_full > min_bytes_allocd(); } GC_VERBOSE_LOG_PRINTF("Immediately reclaimed %ld bytes,heapsize:" " %lu bytes" IF_USE_MUNMAP(" (%lu unmapped)")"\n", (long)GC_bytes_found, (unsigned long)GC_heapsize COMMA_IF_USE_MUNMAP((unsigned long) GC_unmapped_bytes)); GC_n_attempts=0; GC_is_full_gc=FALSE; GC_bytes_allocd_before_gc+=GC_bytes_allocd; GC_non_gc_bytes_at_gc=GC_non_gc_bytes; GC_bytes_allocd=0; GC_bytes_dropped=0; GC_bytes_freed=0; GC_finalizer_bytes_freed=0; IF_USE_MUNMAP(GC_unmap_old()); if (GC_on_collection_event) GC_on_collection_event(GC_EVENT_RECLAIM_END); #ifndef NO_CLOCK if (GC_print_stats){ CLOCK_TYPE done_time; GET_TIME(done_time); #if!defined(SMALL_CONFIG)&&!defined(GC_NO_FINALIZATION) GC_print_finalization_stats(); #endif GC_log_printf("Finalize and initiate sweep took %lu ms %lu ns" "+%lu ms %lu ns\n", MS_TIME_DIFF(finalize_time,start_time), NS_FRAC_TIME_DIFF(finalize_time,start_time), MS_TIME_DIFF(done_time,finalize_time), NS_FRAC_TIME_DIFF(done_time,finalize_time)); } #elif!defined(SMALL_CONFIG)&&!defined(GC_NO_FINALIZATION) if (GC_print_stats) GC_print_finalization_stats(); #endif } STATIC GC_bool GC_try_to_collect_general(GC_stop_func stop_func, GC_bool force_unmap GC_ATTR_UNUSED) { GC_bool result; IF_USE_MUNMAP(int old_unmap_threshold;) IF_CANCEL(int cancel_state;) DCL_LOCK_STATE; if (!EXPECT(GC_is_initialized,TRUE))GC_init(); if (GC_debugging_started)GC_print_all_smashed(); GC_INVOKE_FINALIZERS(); LOCK(); DISABLE_CANCEL(cancel_state); #ifdef USE_MUNMAP old_unmap_threshold=GC_unmap_threshold; if (force_unmap|| (GC_force_unmap_on_gcollect&&old_unmap_threshold > 0)) GC_unmap_threshold=1; #endif ENTER_GC(); GC_noop6(0,0,0,0,0,0); result=GC_try_to_collect_inner(stop_func!=0?stop_func: GC_default_stop_func); EXIT_GC(); IF_USE_MUNMAP(GC_unmap_threshold=old_unmap_threshold); RESTORE_CANCEL(cancel_state); UNLOCK(); if (result){ if (GC_debugging_started)GC_print_all_smashed(); GC_INVOKE_FINALIZERS(); } return(result); } GC_API int GC_CALL GC_try_to_collect(GC_stop_func stop_func) { GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func)); return (int)GC_try_to_collect_general(stop_func,FALSE); } GC_API void GC_CALL GC_gcollect(void) { (void)GC_try_to_collect_general(0,FALSE); if (GC_have_errors)GC_print_all_errors(); } STATIC word GC_heapsize_at_forced_unmap=0; GC_API void GC_CALL GC_gcollect_and_unmap(void) { GC_heapsize_at_forced_unmap=GC_heapsize; (void)GC_try_to_collect_general(GC_never_stop_func,TRUE); } #ifdef USE_PROC_FOR_LIBRARIES GC_INNER void GC_add_to_our_memory(ptr_t p,size_t bytes) { if (0==p)return; if (GC_n_memory>=MAX_HEAP_SECTS) ABORT("Too many GC-allocated memory sections:Increase MAX_HEAP_SECTS"); GC_our_memory[GC_n_memory].hs_start=p; GC_our_memory[GC_n_memory].hs_bytes=bytes; GC_n_memory++; } #endif GC_INNER void GC_add_to_heap(struct hblk*p,size_t bytes) { hdr*phdr; word endp; if (GC_n_heap_sects>=MAX_HEAP_SECTS){ ABORT("Too many heap sections:Increase MAXHINCR or MAX_HEAP_SECTS"); } while ((word)p<=HBLKSIZE){ ++p; bytes-=HBLKSIZE; if (0==bytes)return; } endp=(word)p+bytes; if (endp<=(word)p){ bytes-=HBLKSIZE; if (0==bytes)return; endp-=HBLKSIZE; } phdr=GC_install_header(p); if (0==phdr){ return; } GC_ASSERT(endp > (word)p&&endp==(word)p+bytes); GC_heap_sects[GC_n_heap_sects].hs_start=(ptr_t)p; GC_heap_sects[GC_n_heap_sects].hs_bytes=bytes; GC_n_heap_sects++; phdr->hb_sz=bytes; phdr->hb_flags=0; GC_freehblk(p); GC_heapsize+=bytes; GC_collect_at_heapsize+=bytes; if (GC_collect_at_heapsize < GC_heapsize) GC_collect_at_heapsize=GC_WORD_MAX; if ((word)p<=(word)GC_least_plausible_heap_addr ||GC_least_plausible_heap_addr==0){ GC_least_plausible_heap_addr=(void*)((ptr_t)p - sizeof(word)); } if ((word)p+bytes>=(word)GC_greatest_plausible_heap_addr){ GC_greatest_plausible_heap_addr=(void*)endp; } } #if!defined(NO_DEBUGGING) void GC_print_heap_sects(void) { unsigned i; GC_printf("Total heap size:%lu" IF_USE_MUNMAP(" (%lu unmapped)")"\n", (unsigned long)GC_heapsize COMMA_IF_USE_MUNMAP((unsigned long)GC_unmapped_bytes)); for (i=0;i < GC_n_heap_sects;i++){ ptr_t start=GC_heap_sects[i].hs_start; size_t len=GC_heap_sects[i].hs_bytes; struct hblk*h; unsigned nbl=0; for (h=(struct hblk*)start;(word)h < (word)(start+len);h++){ if (GC_is_black_listed(h,HBLKSIZE))nbl++; } GC_printf("Section %d from %p to %p %u/%lu blacklisted\n", i,(void*)start,(void*)&start[len], nbl,(unsigned long)divHBLKSZ(len)); } } #endif void*GC_least_plausible_heap_addr=(void*)GC_WORD_MAX; void*GC_greatest_plausible_heap_addr=0; GC_INLINE word GC_max(word x,word y) { return(x > y?x:y); } GC_INLINE word GC_min(word x,word y) { return(x < y?x:y); } STATIC word GC_max_heapsize=0; GC_API void GC_CALL GC_set_max_heap_size(GC_word n) { GC_max_heapsize=n; } GC_word GC_max_retries=0; GC_INNER GC_bool GC_expand_hp_inner(word n) { size_t bytes; struct hblk*space; word expansion_slop; GC_ASSERT(I_HOLD_LOCK()); GC_ASSERT(GC_page_size!=0); if (n < MINHINCR)n=MINHINCR; bytes=ROUNDUP_PAGESIZE((size_t)n*HBLKSIZE); if (GC_max_heapsize!=0 &&(GC_max_heapsize < (word)bytes ||GC_heapsize > GC_max_heapsize - (word)bytes)){ return(FALSE); } space=GET_MEM(bytes); GC_add_to_our_memory((ptr_t)space,bytes); if (space==0){ WARN("Failed to expand heap by %" WARN_PRIdPTR " bytes\n", (word)bytes); return(FALSE); } GC_INFOLOG_PRINTF("Grow heap to %lu KiB after %lu bytes allocated\n", TO_KiB_UL(GC_heapsize+(word)bytes), (unsigned long)GC_bytes_allocd); expansion_slop=min_bytes_allocd()+4*MAXHINCR*HBLKSIZE; if ((GC_last_heap_addr==0&&!((word)space&SIGNB)) ||(GC_last_heap_addr!=0 &&(word)GC_last_heap_addr < (word)space)){ word new_limit=(word)space+(word)bytes+expansion_slop; if (new_limit > (word)space){ GC_greatest_plausible_heap_addr= (void*)GC_max((word)GC_greatest_plausible_heap_addr, (word)new_limit); } } else { word new_limit=(word)space - expansion_slop; if (new_limit < (word)space){ GC_least_plausible_heap_addr= (void*)GC_min((word)GC_least_plausible_heap_addr, (word)space - expansion_slop); } } GC_prev_heap_addr=GC_last_heap_addr; GC_last_heap_addr=(ptr_t)space; GC_add_to_heap(space,bytes); GC_collect_at_heapsize= GC_heapsize+expansion_slop - 2*MAXHINCR*HBLKSIZE; if (GC_collect_at_heapsize < GC_heapsize) GC_collect_at_heapsize=GC_WORD_MAX; if (GC_on_heap_resize) (*GC_on_heap_resize)(GC_heapsize); return(TRUE); } GC_API int GC_CALL GC_expand_hp(size_t bytes) { int result; DCL_LOCK_STATE; if (!EXPECT(GC_is_initialized,TRUE))GC_init(); LOCK(); result=(int)GC_expand_hp_inner(divHBLKSZ((word)bytes)); if (result)GC_requested_heapsize+=bytes; UNLOCK(); return(result); } GC_INNER unsigned GC_fail_count=0; #if defined(GC_ALLOCD_BYTES_PER_FINALIZER)&&!defined(CPPCHECK) STATIC word GC_allocd_bytes_per_finalizer=GC_ALLOCD_BYTES_PER_FINALIZER; #else STATIC word GC_allocd_bytes_per_finalizer=10000; #endif GC_API void GC_CALL GC_set_allocd_bytes_per_finalizer(GC_word value) { GC_allocd_bytes_per_finalizer=value; } GC_API GC_word GC_CALL GC_get_allocd_bytes_per_finalizer(void) { return GC_allocd_bytes_per_finalizer; } static word last_fo_entries=0; static word last_bytes_finalized=0; GC_INNER GC_bool GC_collect_or_expand(word needed_blocks, GC_bool ignore_off_page, GC_bool retry) { GC_bool gc_not_stopped=TRUE; word blocks_to_get; IF_CANCEL(int cancel_state;) GC_ASSERT(I_HOLD_LOCK()); DISABLE_CANCEL(cancel_state); if (!GC_incremental&&!GC_dont_gc&& ((GC_dont_expand&&GC_bytes_allocd > 0) ||(GC_fo_entries > last_fo_entries &&(last_bytes_finalized|GC_bytes_finalized)!=0 &&(GC_fo_entries - last_fo_entries) *GC_allocd_bytes_per_finalizer > GC_bytes_allocd) ||GC_should_collect())){ gc_not_stopped=GC_try_to_collect_inner( GC_bytes_allocd > 0&&(!GC_dont_expand||!retry)? GC_default_stop_func:GC_never_stop_func); if (gc_not_stopped==TRUE||!retry){ last_fo_entries=GC_fo_entries; last_bytes_finalized=GC_bytes_finalized; RESTORE_CANCEL(cancel_state); return(TRUE); } } blocks_to_get=(GC_heapsize - GC_heapsize_at_forced_unmap) /(HBLKSIZE*GC_free_space_divisor) +needed_blocks; if (blocks_to_get > MAXHINCR){ word slop; if (ignore_off_page){ slop=4; } else { slop=2*divHBLKSZ(BL_LIMIT); if (slop > needed_blocks)slop=needed_blocks; } if (needed_blocks+slop > MAXHINCR){ blocks_to_get=needed_blocks+slop; } else { blocks_to_get=MAXHINCR; } if (blocks_to_get > divHBLKSZ(GC_WORD_MAX)) blocks_to_get=divHBLKSZ(GC_WORD_MAX); } if (!GC_expand_hp_inner(blocks_to_get) &&(blocks_to_get==needed_blocks ||!GC_expand_hp_inner(needed_blocks))){ if (gc_not_stopped==FALSE){ GC_gcollect_inner(); GC_ASSERT(GC_bytes_allocd==0); } else if (GC_fail_count++< GC_max_retries){ WARN("Out of Memory!Trying to continue...\n",0); GC_gcollect_inner(); } else { #if!defined(AMIGA)||!defined(GC_AMIGA_FASTALLOC) WARN("Out of Memory!Heap size:%" WARN_PRIdPTR " MiB." " Returning NULL!\n",(GC_heapsize - GC_unmapped_bytes)>>20); #endif RESTORE_CANCEL(cancel_state); return(FALSE); } } else if (GC_fail_count){ GC_COND_LOG_PRINTF("Memory available again...\n"); } RESTORE_CANCEL(cancel_state); return(TRUE); } GC_INNER ptr_t GC_allocobj(size_t gran,int kind) { void**flh=&(GC_obj_kinds[kind].ok_freelist[gran]); GC_bool tried_minor=FALSE; GC_bool retry=FALSE; GC_ASSERT(I_HOLD_LOCK()); if (gran==0)return(0); while (*flh==0){ ENTER_GC(); #ifndef GC_DISABLE_INCREMENTAL if (GC_incremental&&GC_time_limit!=GC_TIME_UNLIMITED){ GC_collect_a_little_inner(1); } #endif GC_ASSERT(!GC_is_full_gc ||NULL==GC_obj_kinds[kind].ok_reclaim_list ||NULL==GC_obj_kinds[kind].ok_reclaim_list[gran]); GC_continue_reclaim(gran,kind); EXIT_GC(); #if defined(CPPCHECK) GC_noop1((word)&flh); #endif if (NULL==*flh){ GC_new_hblk(gran,kind); #if defined(CPPCHECK) GC_noop1((word)&flh); #endif if (NULL==*flh){ ENTER_GC(); if (GC_incremental&&GC_time_limit==GC_TIME_UNLIMITED &&!tried_minor){ GC_collect_a_little_inner(1); tried_minor=TRUE; } else { if (!GC_collect_or_expand(1,FALSE,retry)){ EXIT_GC(); return(0); } retry=TRUE; } EXIT_GC(); } } } GC_fail_count=0; return (ptr_t)(*flh); } #ifndef MSWINCE #include #endif #include #ifndef SHORT_DBG_HDRS GC_INNER int GC_has_other_debug_info(ptr_t p) { ptr_t body=(ptr_t)((oh*)p+1); word sz=GC_size(p); if (HBLKPTR(p)!=HBLKPTR((ptr_t)body) ||sz < DEBUG_BYTES+EXTRA_BYTES){ return 0; } if (((oh*)p)->oh_sf!=(START_FLAG^(word)body) &&((word*)p)[BYTES_TO_WORDS(sz)-1]!=(END_FLAG^(word)body)){ return 0; } if (((oh*)p)->oh_sz==sz){ return -1; } return 1; } #endif #ifdef LINT2 long GC_random(void) { static unsigned seed=1; seed=(seed*1103515245U+12345)&GC_RAND_MAX; return (long)seed; } #endif #ifdef KEEP_BACK_PTRS #ifdef LINT2 #define RANDOM()GC_random() #else #include #define GC_RAND_MAX RAND_MAX #if defined(__GLIBC__)||defined(SOLARIS)||defined(HPUX)||defined(IRIX5)||defined(OSF1) #define RANDOM()random() #else #define RANDOM()(long)rand() #endif #endif GC_INNER void GC_store_back_pointer(ptr_t source,ptr_t dest) { if (GC_HAS_DEBUG_INFO(dest)){ #ifdef PARALLEL_MARK AO_store((volatile AO_t*)&((oh*)dest)->oh_back_ptr, (AO_t)HIDE_BACK_PTR(source)); #else ((oh*)dest)->oh_back_ptr=HIDE_BACK_PTR(source); #endif } } GC_INNER void GC_marked_for_finalization(ptr_t dest) { GC_store_back_pointer(MARKED_FOR_FINALIZATION,dest); } GC_API GC_ref_kind GC_CALL GC_get_back_ptr_info(void*dest,void**base_p, size_t*offset_p) { oh*hdr=(oh*)GC_base(dest); ptr_t bp; ptr_t bp_base; #ifdef LINT2 if (!hdr)ABORT("Invalid GC_get_back_ptr_info argument"); #endif if (!GC_HAS_DEBUG_INFO((ptr_t)hdr))return GC_NO_SPACE; bp=(ptr_t)GC_REVEAL_POINTER(hdr->oh_back_ptr); if (MARKED_FOR_FINALIZATION==bp)return GC_FINALIZER_REFD; if (MARKED_FROM_REGISTER==bp)return GC_REFD_FROM_REG; if (NOT_MARKED==bp)return GC_UNREFERENCED; #if ALIGNMENT==1 { ptr_t alternate_ptr=bp+1; ptr_t target=*(ptr_t*)bp; ptr_t alternate_target=*(ptr_t*)alternate_ptr; if ((word)alternate_target>=(word)GC_least_plausible_heap_addr &&(word)alternate_target<=(word)GC_greatest_plausible_heap_addr &&((word)target < (word)GC_least_plausible_heap_addr ||(word)target > (word)GC_greatest_plausible_heap_addr)){ bp=alternate_ptr; } } #endif bp_base=(ptr_t)GC_base(bp); if (NULL==bp_base){ *base_p=bp; *offset_p=0; return GC_REFD_FROM_ROOT; } else { if (GC_HAS_DEBUG_INFO(bp_base))bp_base+=sizeof(oh); *base_p=bp_base; *offset_p=bp - bp_base; return GC_REFD_FROM_HEAP; } } GC_API void*GC_CALL GC_generate_random_heap_address(void) { size_t i; word heap_offset=RANDOM(); if (GC_heapsize > GC_RAND_MAX){ heap_offset*=GC_RAND_MAX; heap_offset+=RANDOM(); } heap_offset%=GC_heapsize; for (i=0;;++i){ size_t size; if (i>=GC_n_heap_sects) ABORT("GC_generate_random_heap_address:size inconsistency"); size=GC_heap_sects[i].hs_bytes; if (heap_offset < size){ break; } else { heap_offset-=size; } } return GC_heap_sects[i].hs_start+heap_offset; } GC_API void*GC_CALL GC_generate_random_valid_address(void) { ptr_t result; ptr_t base; do { result=(ptr_t)GC_generate_random_heap_address(); base=(ptr_t)GC_base(result); } while (NULL==base||!GC_is_marked(base)); return result; } GC_API void GC_CALL GC_print_backtrace(void*p) { void*current=p; int i; GC_ref_kind source; size_t offset; void*base; GC_print_heap_obj((ptr_t)GC_base(current)); for (i=0;;++i){ source=GC_get_back_ptr_info(current,&base,&offset); if (GC_UNREFERENCED==source){ GC_err_printf("Reference could not be found\n"); goto out; } if (GC_NO_SPACE==source){ GC_err_printf("No debug info in object:Can't find reference\n"); goto out; } GC_err_printf("Reachable via %d levels of pointers from ",i); switch(source){ case GC_REFD_FROM_ROOT: GC_err_printf("root at %p\n\n",base); goto out; case GC_REFD_FROM_REG: GC_err_printf("root in register\n\n"); goto out; case GC_FINALIZER_REFD: GC_err_printf("list of finalizable objects\n\n"); goto out; case GC_REFD_FROM_HEAP: GC_err_printf("offset %ld in object:\n",(long)offset); GC_print_heap_obj((ptr_t)GC_base(base)); break; default: GC_err_printf("INTERNAL ERROR:UNEXPECTED SOURCE!!!!\n"); goto out; } current=base; } out:; } GC_INNER void GC_generate_random_backtrace_no_gc(void) { void*current; current=GC_generate_random_valid_address(); GC_printf("\n****Chosen address %p in object\n",current); GC_print_backtrace(current); } GC_API void GC_CALL GC_generate_random_backtrace(void) { if (GC_try_to_collect(GC_never_stop_func)==0){ GC_err_printf("Cannot generate a backtrace:" "garbage collection is disabled!\n"); return; } GC_generate_random_backtrace_no_gc(); } #endif #define CROSSES_HBLK(p,sz)(((word)((p)+sizeof(oh)+(sz)- 1)^(word)(p))>=HBLKSIZE) GC_INNER void*GC_store_debug_info_inner(void*p,word sz GC_ATTR_UNUSED, const char*string,int linenum) { word*result=(word*)((oh*)p+1); GC_ASSERT(I_HOLD_LOCK()); GC_ASSERT(GC_size(p)>=sizeof(oh)+sz); GC_ASSERT(!(SMALL_OBJ(sz)&&CROSSES_HBLK((ptr_t)p,sz))); #ifdef KEEP_BACK_PTRS ((oh*)p)->oh_back_ptr=HIDE_BACK_PTR(NOT_MARKED); #endif #ifdef MAKE_BACK_GRAPH ((oh*)p)->oh_bg_ptr=HIDE_BACK_PTR((ptr_t)0); #endif ((oh*)p)->oh_string=string; ((oh*)p)->oh_int=linenum; #ifndef SHORT_DBG_HDRS ((oh*)p)->oh_sz=sz; ((oh*)p)->oh_sf=START_FLAG^(word)result; ((word*)p)[BYTES_TO_WORDS(GC_size(p))-1]= result[SIMPLE_ROUNDED_UP_WORDS(sz)]=END_FLAG^(word)result; #endif return result; } static void*store_debug_info(void*p,size_t lb, const char*fn,GC_EXTRA_PARAMS) { void*result; DCL_LOCK_STATE; if (NULL==p){ GC_err_printf("%s(%lu)returning NULL (%s:%d)\n", fn,(unsigned long)lb,s,i); return NULL; } LOCK(); if (!GC_debugging_started) GC_start_debugging_inner(); ADD_CALL_CHAIN(p,ra); result=GC_store_debug_info_inner(p,(word)lb,s,i); UNLOCK(); return result; } #ifndef SHORT_DBG_HDRS STATIC ptr_t GC_check_annotated_obj(oh*ohdr) { ptr_t body=(ptr_t)(ohdr+1); word gc_sz=GC_size((ptr_t)ohdr); if (ohdr->oh_sz+DEBUG_BYTES > gc_sz){ return((ptr_t)(&(ohdr->oh_sz))); } if (ohdr->oh_sf!=(START_FLAG^(word)body)){ return((ptr_t)(&(ohdr->oh_sf))); } if (((word*)ohdr)[BYTES_TO_WORDS(gc_sz)-1]!=(END_FLAG^(word)body)){ return (ptr_t)(&((word*)ohdr)[BYTES_TO_WORDS(gc_sz)-1]); } if (((word*)body)[SIMPLE_ROUNDED_UP_WORDS(ohdr->oh_sz)] !=(END_FLAG^(word)body)){ return (ptr_t)(&((word*)body)[SIMPLE_ROUNDED_UP_WORDS(ohdr->oh_sz)]); } return(0); } #endif STATIC GC_describe_type_fn GC_describe_type_fns[MAXOBJKINDS]={0}; GC_API void GC_CALL GC_register_describe_type_fn(int kind, GC_describe_type_fn fn) { GC_describe_type_fns[kind]=fn; } #define GET_OH_LINENUM(ohdr)((int)(ohdr)->oh_int) #ifndef SHORT_DBG_HDRS #define IF_NOT_SHORTDBG_HDRS(x)x #define COMMA_IFNOT_SHORTDBG_HDRS(x),x #else #define IF_NOT_SHORTDBG_HDRS(x) #define COMMA_IFNOT_SHORTDBG_HDRS(x) #endif STATIC void GC_print_obj(ptr_t p) { oh*ohdr=(oh*)GC_base(p); ptr_t q; hdr*hhdr; int kind; const char*kind_str; char buffer[GC_TYPE_DESCR_LEN+1]; GC_ASSERT(I_DONT_HOLD_LOCK()); #ifdef LINT2 if (!ohdr)ABORT("Invalid GC_print_obj argument"); #endif q=(ptr_t)(ohdr+1); hhdr=GC_find_header(q); kind=hhdr->hb_obj_kind; if (0!=GC_describe_type_fns[kind]&&GC_is_marked(ohdr)){ buffer[GC_TYPE_DESCR_LEN]=0; (GC_describe_type_fns[kind])(q,buffer); GC_ASSERT(buffer[GC_TYPE_DESCR_LEN]==0); kind_str=buffer; } else { switch(kind){ case PTRFREE: kind_str="PTRFREE"; break; case NORMAL: kind_str="NORMAL"; break; case UNCOLLECTABLE: kind_str="UNCOLLECTABLE"; break; #ifdef GC_ATOMIC_UNCOLLECTABLE case AUNCOLLECTABLE: kind_str="ATOMIC_UNCOLLECTABLE"; break; #endif default: kind_str=NULL; } } if (NULL!=kind_str){ GC_err_printf("%p (%s:%d," IF_NOT_SHORTDBG_HDRS(" sz=%lu,")" %s)\n", (void*)((ptr_t)ohdr+sizeof(oh)), ohdr->oh_string,GET_OH_LINENUM(ohdr) COMMA_IFNOT_SHORTDBG_HDRS((unsigned long)ohdr->oh_sz), kind_str); } else { GC_err_printf("%p (%s:%d," IF_NOT_SHORTDBG_HDRS(" sz=%lu,") " kind=%d descr=0x%lx)\n", (void*)((ptr_t)ohdr+sizeof(oh)), ohdr->oh_string,GET_OH_LINENUM(ohdr) COMMA_IFNOT_SHORTDBG_HDRS((unsigned long)ohdr->oh_sz), kind,(unsigned long)hhdr->hb_descr); } PRINT_CALL_CHAIN(ohdr); } STATIC void GC_debug_print_heap_obj_proc(ptr_t p) { GC_ASSERT(I_DONT_HOLD_LOCK()); if (GC_HAS_DEBUG_INFO(p)){ GC_print_obj(p); } else { GC_default_print_heap_obj_proc(p); } } #ifndef SHORT_DBG_HDRS STATIC void GC_print_smashed_obj(const char*msg,void*p, ptr_t clobbered_addr) { oh*ohdr=(oh*)GC_base(p); GC_ASSERT(I_DONT_HOLD_LOCK()); #ifdef LINT2 if (!ohdr)ABORT("Invalid GC_print_smashed_obj argument"); #endif if ((word)clobbered_addr<=(word)(&ohdr->oh_sz) ||ohdr->oh_string==0){ GC_err_printf( "%s %p in or near object at %p(,appr. sz=%lu)\n", msg,(void*)clobbered_addr,p, (unsigned long)(GC_size((ptr_t)ohdr)- DEBUG_BYTES)); } else { GC_err_printf("%s %p in or near object at %p (%s:%d,sz=%lu)\n", msg,(void*)clobbered_addr,p, (word)(ohdr->oh_string)< HBLKSIZE?"(smashed string)": ohdr->oh_string[0]=='\0'?"EMPTY(smashed?)": ohdr->oh_string, GET_OH_LINENUM(ohdr),(unsigned long)(ohdr->oh_sz)); PRINT_CALL_CHAIN(ohdr); } } STATIC void GC_check_heap_proc (void); STATIC void GC_print_all_smashed_proc (void); #else STATIC void GC_do_nothing(void){} #endif GC_INNER void GC_start_debugging_inner(void) { GC_ASSERT(I_HOLD_LOCK()); #ifndef SHORT_DBG_HDRS GC_check_heap=GC_check_heap_proc; GC_print_all_smashed=GC_print_all_smashed_proc; #else GC_check_heap=GC_do_nothing; GC_print_all_smashed=GC_do_nothing; #endif GC_print_heap_obj=GC_debug_print_heap_obj_proc; GC_debugging_started=TRUE; GC_register_displacement_inner((word)sizeof(oh)); #if defined(CPPCHECK) GC_noop1(GC_debug_header_size); #endif } const size_t GC_debug_header_size=sizeof(oh); GC_API size_t GC_CALL GC_get_debug_header_size(void){ return sizeof(oh); } GC_API void GC_CALL GC_debug_register_displacement(size_t offset) { DCL_LOCK_STATE; LOCK(); GC_register_displacement_inner(offset); GC_register_displacement_inner((word)sizeof(oh)+offset); UNLOCK(); } #ifdef GC_ADD_CALLER #if defined(HAVE_DLADDR)&&defined(GC_HAVE_RETURN_ADDR_PARENT) #include STATIC void GC_caller_func_offset(word ad,const char**symp,int*offp) { Dl_info caller; if (ad&&dladdr((void*)ad,&caller)&&caller.dli_sname!=NULL){ *symp=caller.dli_sname; *offp=(int)((char*)ad - (char*)caller.dli_saddr); } if (NULL==*symp){ *symp="unknown"; } } #else #define GC_caller_func_offset(ad,symp,offp)(void)(*(symp)="unknown") #endif #endif GC_API GC_ATTR_MALLOC void*GC_CALL GC_debug_malloc(size_t lb, GC_EXTRA_PARAMS) { void*result; result=GC_malloc(SIZET_SAT_ADD(lb,DEBUG_BYTES)); #ifdef GC_ADD_CALLER if (s==NULL){ GC_caller_func_offset(ra,&s,&i); } #endif return store_debug_info(result,lb,"GC_debug_malloc",OPT_RA s,i); } GC_API GC_ATTR_MALLOC void*GC_CALL GC_debug_malloc_ignore_off_page(size_t lb,GC_EXTRA_PARAMS) { void*result=GC_malloc_ignore_off_page(SIZET_SAT_ADD(lb,DEBUG_BYTES)); return store_debug_info(result,lb,"GC_debug_malloc_ignore_off_page", OPT_RA s,i); } GC_API GC_ATTR_MALLOC void*GC_CALL GC_debug_malloc_atomic_ignore_off_page(size_t lb,GC_EXTRA_PARAMS) { void*result=GC_malloc_atomic_ignore_off_page( SIZET_SAT_ADD(lb,DEBUG_BYTES)); return store_debug_info(result,lb, "GC_debug_malloc_atomic_ignore_off_page", OPT_RA s,i); } STATIC void*GC_debug_generic_malloc(size_t lb,int knd,GC_EXTRA_PARAMS) { void*result=GC_generic_malloc(SIZET_SAT_ADD(lb,DEBUG_BYTES),knd); return store_debug_info(result,lb,"GC_debug_generic_malloc", OPT_RA s,i); } #ifdef DBG_HDRS_ALL GC_INNER void*GC_debug_generic_malloc_inner(size_t lb,int k) { void*result; GC_ASSERT(I_HOLD_LOCK()); result=GC_generic_malloc_inner(SIZET_SAT_ADD(lb,DEBUG_BYTES),k); if (NULL==result){ GC_err_printf("GC internal allocation (%lu bytes)returning NULL\n", (unsigned long)lb); return(0); } if (!GC_debugging_started){ GC_start_debugging_inner(); } ADD_CALL_CHAIN(result,GC_RETURN_ADDR); return (GC_store_debug_info_inner(result,(word)lb,"INTERNAL",0)); } GC_INNER void*GC_debug_generic_malloc_inner_ignore_off_page(size_t lb, int k) { void*result; GC_ASSERT(I_HOLD_LOCK()); result=GC_generic_malloc_inner_ignore_off_page( SIZET_SAT_ADD(lb,DEBUG_BYTES),k); if (NULL==result){ GC_err_printf("GC internal allocation (%lu bytes)returning NULL\n", (unsigned long)lb); return(0); } if (!GC_debugging_started){ GC_start_debugging_inner(); } ADD_CALL_CHAIN(result,GC_RETURN_ADDR); return (GC_store_debug_info_inner(result,(word)lb,"INTERNAL",0)); } #endif #ifndef CPPCHECK GC_API void*GC_CALL GC_debug_malloc_stubborn(size_t lb,GC_EXTRA_PARAMS) { return GC_debug_malloc(lb,OPT_RA s,i); } GC_API void GC_CALL GC_debug_change_stubborn( const void*p GC_ATTR_UNUSED){} #endif GC_API void GC_CALL GC_debug_end_stubborn_change(const void*p) { const void*q=GC_base_C(p); if (NULL==q){ ABORT_ARG1("GC_debug_end_stubborn_change:bad arg",":%p",p); } GC_end_stubborn_change(q); } GC_API void GC_CALL GC_debug_ptr_store_and_dirty(void*p,const void*q) { *(void**)GC_is_visible(p)=GC_is_valid_displacement((void*)q); GC_debug_end_stubborn_change(p); REACHABLE_AFTER_DIRTY(q); } GC_API GC_ATTR_MALLOC void*GC_CALL GC_debug_malloc_atomic(size_t lb, GC_EXTRA_PARAMS) { void*result=GC_malloc_atomic(SIZET_SAT_ADD(lb,DEBUG_BYTES)); return store_debug_info(result,lb,"GC_debug_malloc_atomic", OPT_RA s,i); } GC_API GC_ATTR_MALLOC char*GC_CALL GC_debug_strdup(const char*str, GC_EXTRA_PARAMS) { char*copy; size_t lb; if (str==NULL){ if (GC_find_leak) GC_err_printf("strdup(NULL)behavior is undefined\n"); return NULL; } lb=strlen(str)+1; copy=(char*)GC_debug_malloc_atomic(lb,OPT_RA s,i); if (copy==NULL){ #ifndef MSWINCE errno=ENOMEM; #endif return NULL; } BCOPY(str,copy,lb); return copy; } GC_API GC_ATTR_MALLOC char*GC_CALL GC_debug_strndup(const char*str, size_t size,GC_EXTRA_PARAMS) { char*copy; size_t len=strlen(str); if (len > size) len=size; copy=(char*)GC_debug_malloc_atomic(len+1,OPT_RA s,i); if (copy==NULL){ #ifndef MSWINCE errno=ENOMEM; #endif return NULL; } if (len > 0) BCOPY(str,copy,len); copy[len]='\0'; return copy; } #ifdef GC_REQUIRE_WCSDUP #include GC_API GC_ATTR_MALLOC wchar_t*GC_CALL GC_debug_wcsdup(const wchar_t*str, GC_EXTRA_PARAMS) { size_t lb=(wcslen(str)+1)*sizeof(wchar_t); wchar_t*copy=(wchar_t*)GC_debug_malloc_atomic(lb,OPT_RA s,i); if (copy==NULL){ #ifndef MSWINCE errno=ENOMEM; #endif return NULL; } BCOPY(str,copy,lb); return copy; } #endif GC_API GC_ATTR_MALLOC void*GC_CALL GC_debug_malloc_uncollectable(size_t lb, GC_EXTRA_PARAMS) { void*result=GC_malloc_uncollectable( SIZET_SAT_ADD(lb,UNCOLLECTABLE_DEBUG_BYTES)); return store_debug_info(result,lb,"GC_debug_malloc_uncollectable", OPT_RA s,i); } #ifdef GC_ATOMIC_UNCOLLECTABLE GC_API GC_ATTR_MALLOC void*GC_CALL GC_debug_malloc_atomic_uncollectable(size_t lb,GC_EXTRA_PARAMS) { void*result=GC_malloc_atomic_uncollectable( SIZET_SAT_ADD(lb,UNCOLLECTABLE_DEBUG_BYTES)); return store_debug_info(result,lb, "GC_debug_malloc_atomic_uncollectable", OPT_RA s,i); } #endif #ifndef GC_FREED_MEM_MARKER #if CPP_WORDSZ==32 #define GC_FREED_MEM_MARKER 0xdeadbeef #else #define GC_FREED_MEM_MARKER GC_WORD_C(0xEFBEADDEdeadbeef) #endif #endif GC_API void GC_CALL GC_debug_free(void*p) { ptr_t base; if (0==p)return; base=(ptr_t)GC_base(p); if (NULL==base){ #if defined(REDIRECT_MALLOC)&&((defined(NEED_CALLINFO)&&defined(GC_HAVE_BUILTIN_BACKTRACE))||defined(GC_LINUX_THREADS)||defined(GC_SOLARIS_THREADS)||defined(MSWIN32)) if (!GC_is_heap_ptr(p))return; #endif ABORT_ARG1("Invalid pointer passed to free()",":%p",p); } if ((ptr_t)p - (ptr_t)base!=sizeof(oh)){ #if defined(REDIRECT_FREE)&&defined(USE_PROC_FOR_LIBRARIES) #endif GC_err_printf( "GC_debug_free called on pointer %p w/o debugging info\n",p); } else { #ifndef SHORT_DBG_HDRS ptr_t clobbered=GC_check_annotated_obj((oh*)base); word sz=GC_size(base); if (clobbered!=0){ GC_have_errors=TRUE; if (((oh*)base)->oh_sz==sz){ GC_print_smashed_obj( "GC_debug_free:found previously deallocated (?)object at", p,clobbered); return; } else { GC_print_smashed_obj("GC_debug_free:found smashed location at", p,clobbered); } } ((oh*)base)->oh_sz=sz; #endif } if (GC_find_leak #ifndef SHORT_DBG_HDRS &&((ptr_t)p - (ptr_t)base!=sizeof(oh)||!GC_findleak_delay_free) #endif ){ GC_free(base); } else { hdr*hhdr=HDR(p); if (hhdr->hb_obj_kind==UNCOLLECTABLE #ifdef GC_ATOMIC_UNCOLLECTABLE ||hhdr->hb_obj_kind==AUNCOLLECTABLE #endif ){ GC_free(base); } else { word i; word sz=hhdr->hb_sz; word obj_sz=BYTES_TO_WORDS(sz - sizeof(oh)); for (i=0;i < obj_sz;++i) ((word*)p)[i]=GC_FREED_MEM_MARKER; GC_ASSERT((word*)p+i==(word*)(base+sz)); LOCK(); GC_bytes_freed+=sz; UNLOCK(); } } } #if defined(THREADS)&&defined(DBG_HDRS_ALL) GC_INNER void GC_debug_free_inner(void*p) { ptr_t base=(ptr_t)GC_base(p); GC_ASSERT((ptr_t)p - (ptr_t)base==sizeof(oh)); #ifdef LINT2 if (!base)ABORT("Invalid GC_debug_free_inner argument"); #endif #ifndef SHORT_DBG_HDRS ((oh*)base)->oh_sz=GC_size(base); #endif GC_free_inner(base); } #endif GC_API void*GC_CALL GC_debug_realloc(void*p,size_t lb,GC_EXTRA_PARAMS) { void*base; void*result; hdr*hhdr; if (p==0){ return GC_debug_malloc(lb,OPT_RA s,i); } if (0==lb){ GC_debug_free(p); return NULL; } #ifdef GC_ADD_CALLER if (s==NULL){ GC_caller_func_offset(ra,&s,&i); } #endif base=GC_base(p); if (base==0){ ABORT_ARG1("Invalid pointer passed to realloc()",":%p",p); } if ((ptr_t)p - (ptr_t)base!=sizeof(oh)){ GC_err_printf( "GC_debug_realloc called on pointer %p w/o debugging info\n",p); return(GC_realloc(p,lb)); } hhdr=HDR(base); switch (hhdr->hb_obj_kind){ case NORMAL: result=GC_debug_malloc(lb,OPT_RA s,i); break; case PTRFREE: result=GC_debug_malloc_atomic(lb,OPT_RA s,i); break; case UNCOLLECTABLE: result=GC_debug_malloc_uncollectable(lb,OPT_RA s,i); break; #ifdef GC_ATOMIC_UNCOLLECTABLE case AUNCOLLECTABLE: result=GC_debug_malloc_atomic_uncollectable(lb,OPT_RA s,i); break; #endif default: result=NULL; ABORT_RET("GC_debug_realloc:encountered bad kind"); } if (result!=NULL){ size_t old_sz; #ifdef SHORT_DBG_HDRS old_sz=GC_size(base)- sizeof(oh); #else old_sz=((oh*)base)->oh_sz; #endif if (old_sz > 0) BCOPY(p,result,old_sz < lb?old_sz:lb); GC_debug_free(p); } return(result); } GC_API GC_ATTR_MALLOC void*GC_CALL GC_debug_generic_or_special_malloc(size_t lb,int knd,GC_EXTRA_PARAMS) { switch (knd){ case PTRFREE: return GC_debug_malloc_atomic(lb,OPT_RA s,i); case NORMAL: return GC_debug_malloc(lb,OPT_RA s,i); case UNCOLLECTABLE: return GC_debug_malloc_uncollectable(lb,OPT_RA s,i); #ifdef GC_ATOMIC_UNCOLLECTABLE case AUNCOLLECTABLE: return GC_debug_malloc_atomic_uncollectable(lb,OPT_RA s,i); #endif default: return GC_debug_generic_malloc(lb,knd,OPT_RA s,i); } } #ifndef SHORT_DBG_HDRS #ifndef MAX_SMASHED #define MAX_SMASHED 20 #endif STATIC ptr_t GC_smashed[MAX_SMASHED]={0}; STATIC unsigned GC_n_smashed=0; STATIC void GC_add_smashed(ptr_t smashed) { GC_ASSERT(GC_is_marked(GC_base(smashed))); GC_smashed[GC_n_smashed]=smashed; if (GC_n_smashed < MAX_SMASHED - 1)++GC_n_smashed; GC_have_errors=TRUE; } STATIC void GC_print_all_smashed_proc(void) { unsigned i; GC_ASSERT(I_DONT_HOLD_LOCK()); if (GC_n_smashed==0)return; GC_err_printf("GC_check_heap_block:found %u smashed heap objects:\n", GC_n_smashed); for (i=0;i < GC_n_smashed;++i){ ptr_t base=(ptr_t)GC_base(GC_smashed[i]); #ifdef LINT2 if (!base)ABORT("Invalid GC_smashed element"); #endif GC_print_smashed_obj("",base+sizeof(oh),GC_smashed[i]); GC_smashed[i]=0; } GC_n_smashed=0; } STATIC void GC_check_heap_block(struct hblk*hbp,word dummy GC_ATTR_UNUSED) { struct hblkhdr*hhdr=HDR(hbp); word sz=hhdr->hb_sz; word bit_no; char*p,*plim; p=hbp->hb_body; if (sz > MAXOBJBYTES){ plim=p; } else { plim=hbp->hb_body+HBLKSIZE - sz; } for (bit_no=0;(word)p<=(word)plim; bit_no+=MARK_BIT_OFFSET(sz),p+=sz){ if (mark_bit_from_hdr(hhdr,bit_no)&&GC_HAS_DEBUG_INFO((ptr_t)p)){ ptr_t clobbered=GC_check_annotated_obj((oh*)p); if (clobbered!=0) GC_add_smashed(clobbered); } } } STATIC void GC_check_heap_proc(void) { GC_STATIC_ASSERT((sizeof(oh)&(GRANULE_BYTES - 1))==0); GC_apply_to_all_blocks(GC_check_heap_block,0); } GC_INNER GC_bool GC_check_leaked(ptr_t base) { word i; word obj_sz; word*p; if ( #if defined(KEEP_BACK_PTRS)||defined(MAKE_BACK_GRAPH) (*(word*)base&1)!=0&& #endif GC_has_other_debug_info(base)>=0) return TRUE; p=(word*)(base+sizeof(oh)); obj_sz=BYTES_TO_WORDS(HDR(base)->hb_sz - sizeof(oh)); for (i=0;i < obj_sz;++i) if (p[i]!=GC_FREED_MEM_MARKER){ GC_set_mark_bit(base); GC_add_smashed((ptr_t)(&p[i])); break; } return FALSE; } #endif #ifndef GC_NO_FINALIZATION struct closure { GC_finalization_proc cl_fn; void*cl_data; }; STATIC void*GC_make_closure(GC_finalization_proc fn,void*data) { struct closure*result= #ifdef DBG_HDRS_ALL (struct closure*)GC_debug_malloc(sizeof (struct closure), GC_EXTRAS); #else (struct closure*)GC_malloc(sizeof (struct closure)); #endif if (result!=0){ result->cl_fn=fn; result->cl_data=data; } return((void*)result); } STATIC void GC_CALLBACK GC_debug_invoke_finalizer(void*obj,void*data) { struct closure*cl=(struct closure*)data; (*(cl->cl_fn))((void*)((char*)obj+sizeof(oh)),cl->cl_data); } #define OFN_UNSET ((GC_finalization_proc)~(signed_word)0) static void store_old(void*obj,GC_finalization_proc my_old_fn, struct closure*my_old_cd,GC_finalization_proc*ofn, void**ocd) { if (0!=my_old_fn){ if (my_old_fn==OFN_UNSET){ return; } if (my_old_fn!=GC_debug_invoke_finalizer){ GC_err_printf("Debuggable object at %p had a non-debug finalizer\n", obj); } else { if (ofn)*ofn=my_old_cd->cl_fn; if (ocd)*ocd=my_old_cd->cl_data; } } else { if (ofn)*ofn=0; if (ocd)*ocd=0; } } GC_API void GC_CALL GC_debug_register_finalizer(void*obj, GC_finalization_proc fn, void*cd,GC_finalization_proc*ofn, void**ocd) { GC_finalization_proc my_old_fn=OFN_UNSET; void*my_old_cd; ptr_t base=(ptr_t)GC_base(obj); if (NULL==base){ if (ocd)*ocd=0; if (ofn)*ofn=0; return; } if ((ptr_t)obj - base!=sizeof(oh)){ GC_err_printf("GC_debug_register_finalizer called with" " non-base-pointer %p\n",obj); } if (0==fn){ GC_register_finalizer(base,0,0,&my_old_fn,&my_old_cd); } else { cd=GC_make_closure(fn,cd); if (cd==0)return; GC_register_finalizer(base,GC_debug_invoke_finalizer, cd,&my_old_fn,&my_old_cd); } store_old(obj,my_old_fn,(struct closure*)my_old_cd,ofn,ocd); } GC_API void GC_CALL GC_debug_register_finalizer_no_order (void*obj,GC_finalization_proc fn, void*cd,GC_finalization_proc*ofn, void**ocd) { GC_finalization_proc my_old_fn=OFN_UNSET; void*my_old_cd; ptr_t base=(ptr_t)GC_base(obj); if (NULL==base){ if (ocd)*ocd=0; if (ofn)*ofn=0; return; } if ((ptr_t)obj - base!=sizeof(oh)){ GC_err_printf("GC_debug_register_finalizer_no_order called with" " non-base-pointer %p\n",obj); } if (0==fn){ GC_register_finalizer_no_order(base,0,0,&my_old_fn,&my_old_cd); } else { cd=GC_make_closure(fn,cd); if (cd==0)return; GC_register_finalizer_no_order(base,GC_debug_invoke_finalizer, cd,&my_old_fn,&my_old_cd); } store_old(obj,my_old_fn,(struct closure*)my_old_cd,ofn,ocd); } GC_API void GC_CALL GC_debug_register_finalizer_unreachable (void*obj,GC_finalization_proc fn, void*cd,GC_finalization_proc*ofn, void**ocd) { GC_finalization_proc my_old_fn=OFN_UNSET; void*my_old_cd; ptr_t base=(ptr_t)GC_base(obj); if (NULL==base){ if (ocd)*ocd=0; if (ofn)*ofn=0; return; } if ((ptr_t)obj - base!=sizeof(oh)){ GC_err_printf("GC_debug_register_finalizer_unreachable called with" " non-base-pointer %p\n",obj); } if (0==fn){ GC_register_finalizer_unreachable(base,0,0,&my_old_fn,&my_old_cd); } else { cd=GC_make_closure(fn,cd); if (cd==0)return; GC_register_finalizer_unreachable(base,GC_debug_invoke_finalizer, cd,&my_old_fn,&my_old_cd); } store_old(obj,my_old_fn,(struct closure*)my_old_cd,ofn,ocd); } GC_API void GC_CALL GC_debug_register_finalizer_ignore_self (void*obj,GC_finalization_proc fn, void*cd,GC_finalization_proc*ofn, void**ocd) { GC_finalization_proc my_old_fn=OFN_UNSET; void*my_old_cd; ptr_t base=(ptr_t)GC_base(obj); if (NULL==base){ if (ocd)*ocd=0; if (ofn)*ofn=0; return; } if ((ptr_t)obj - base!=sizeof(oh)){ GC_err_printf("GC_debug_register_finalizer_ignore_self called with" " non-base-pointer %p\n",obj); } if (0==fn){ GC_register_finalizer_ignore_self(base,0,0,&my_old_fn,&my_old_cd); } else { cd=GC_make_closure(fn,cd); if (cd==0)return; GC_register_finalizer_ignore_self(base,GC_debug_invoke_finalizer, cd,&my_old_fn,&my_old_cd); } store_old(obj,my_old_fn,(struct closure*)my_old_cd,ofn,ocd); } #endif GC_API GC_ATTR_MALLOC void*GC_CALL GC_debug_malloc_replacement(size_t lb) { return GC_debug_malloc(lb,GC_DBG_EXTRAS); } GC_API void*GC_CALL GC_debug_realloc_replacement(void*p,size_t lb) { return GC_debug_realloc(p,lb,GC_DBG_EXTRAS); } #ifndef GC_NO_FINALIZATION #ifndef GC_JAVAXFC_H #define GC_JAVAXFC_H #ifndef GC_H #endif #ifdef __cplusplus extern "C" { #endif GC_API void GC_CALL GC_finalize_all(void); #ifdef GC_THREADS #ifndef GC_SUSPEND_THREAD_ID #define GC_SUSPEND_THREAD_ID void* #endif GC_API void GC_CALL GC_suspend_thread(GC_SUSPEND_THREAD_ID); GC_API void GC_CALL GC_resume_thread(GC_SUSPEND_THREAD_ID); GC_API int GC_CALL GC_is_thread_suspended(GC_SUSPEND_THREAD_ID); #endif #ifdef __cplusplus } #endif #endif typedef void (*finalization_mark_proc)(ptr_t); #define HASH3(addr,size,log_size)((((word)(addr)>>3)^((word)(addr)>>(3+(log_size))))&((size)- 1)) #define HASH2(addr,log_size)HASH3(addr,(word)1<<(log_size),log_size) struct hash_chain_entry { word hidden_key; struct hash_chain_entry*next; }; struct disappearing_link { struct hash_chain_entry prolog; #define dl_hidden_link prolog.hidden_key #define dl_next(x)(struct disappearing_link*)((x)->prolog.next) #define dl_set_next(x,y)(void)((x)->prolog.next=(struct hash_chain_entry*)(y)) word dl_hidden_obj; }; struct finalizable_object { struct hash_chain_entry prolog; #define fo_hidden_base prolog.hidden_key #define fo_next(x)(struct finalizable_object*)((x)->prolog.next) #define fo_set_next(x,y)((x)->prolog.next=(struct hash_chain_entry*)(y)) GC_finalization_proc fo_fn; ptr_t fo_client_data; word fo_object_size; finalization_mark_proc fo_mark_proc; }; #ifdef AO_HAVE_store #define SET_FINALIZE_NOW(fo)AO_store((volatile AO_t*)&GC_fnlz_roots.finalize_now,(AO_t)(fo)) #else #define SET_FINALIZE_NOW(fo)(void)(GC_fnlz_roots.finalize_now=(fo)) #endif GC_API void GC_CALL GC_push_finalizer_structures(void) { GC_ASSERT((word)(&GC_dl_hashtbl.head)% sizeof(word)==0); GC_ASSERT((word)(&GC_fnlz_roots)% sizeof(word)==0); #ifndef GC_LONG_REFS_NOT_NEEDED GC_ASSERT((word)(&GC_ll_hashtbl.head)% sizeof(word)==0); GC_PUSH_ALL_SYM(GC_ll_hashtbl.head); #endif GC_PUSH_ALL_SYM(GC_dl_hashtbl.head); GC_PUSH_ALL_SYM(GC_fnlz_roots); } #ifndef GC_ON_GROW_LOG_SIZE_MIN #define GC_ON_GROW_LOG_SIZE_MIN CPP_LOG_HBLKSIZE #endif STATIC void GC_grow_table(struct hash_chain_entry***table, unsigned*log_size_ptr,word*entries_ptr) { word i; struct hash_chain_entry*p; unsigned log_old_size=*log_size_ptr; unsigned log_new_size=log_old_size+1; word old_size=*table==NULL?0:(word)1<=GC_ON_GROW_LOG_SIZE_MIN&&!GC_incremental){ IF_CANCEL(int cancel_state;) DISABLE_CANCEL(cancel_state); (void)GC_try_to_collect_inner(GC_never_stop_func); RESTORE_CANCEL(cancel_state); if (*entries_ptr < ((word)1<>2)) return; } new_table=(struct hash_chain_entry**) GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE( (size_t)new_size*sizeof(struct hash_chain_entry*), NORMAL); if (new_table==0){ if (*table==0){ ABORT("Insufficient space for initial table allocation"); } else { return; } } for (i=0;i < old_size;i++){ p=(*table)[i]; while (p!=0){ ptr_t real_key=(ptr_t)GC_REVEAL_POINTER(p->hidden_key); struct hash_chain_entry*next=p->next; size_t new_hash=HASH3(real_key,new_size,log_new_size); p->next=new_table[new_hash]; GC_dirty(p); new_table[new_hash]=p; p=next; } } *log_size_ptr=log_new_size; *table=new_table; GC_dirty(new_table); } GC_API int GC_CALL GC_register_disappearing_link(void**link) { ptr_t base; base=(ptr_t)GC_base(link); if (base==0) ABORT("Bad arg to GC_register_disappearing_link"); return(GC_general_register_disappearing_link(link,base)); } STATIC int GC_register_disappearing_link_inner( struct dl_hashtbl_s*dl_hashtbl,void**link, const void*obj,const char*tbl_log_name) { struct disappearing_link*curr_dl; size_t index; struct disappearing_link*new_dl; DCL_LOCK_STATE; if (EXPECT(GC_find_leak,FALSE))return GC_UNIMPLEMENTED; LOCK(); GC_ASSERT(obj!=NULL&&GC_base_C(obj)==obj); if (EXPECT(NULL==dl_hashtbl->head,FALSE) ||EXPECT(dl_hashtbl->entries > ((word)1<log_size),FALSE)){ GC_grow_table((struct hash_chain_entry***)&dl_hashtbl->head, &dl_hashtbl->log_size,&dl_hashtbl->entries); GC_COND_LOG_PRINTF("Grew %s table to %u entries\n",tbl_log_name, 1U<log_size); } index=HASH2(link,dl_hashtbl->log_size); for (curr_dl=dl_hashtbl->head[index];curr_dl!=0; curr_dl=dl_next(curr_dl)){ if (curr_dl->dl_hidden_link==GC_HIDE_POINTER(link)){ curr_dl->dl_hidden_obj=GC_HIDE_POINTER(obj); UNLOCK(); return GC_DUPLICATE; } } new_dl=(struct disappearing_link*) GC_INTERNAL_MALLOC(sizeof(struct disappearing_link),NORMAL); if (0==new_dl){ GC_oom_func oom_fn=GC_oom_fn; UNLOCK(); new_dl=(struct disappearing_link*) (*oom_fn)(sizeof(struct disappearing_link)); if (0==new_dl){ return GC_NO_MEMORY; } LOCK(); index=HASH2(link,dl_hashtbl->log_size); for (curr_dl=dl_hashtbl->head[index];curr_dl!=0; curr_dl=dl_next(curr_dl)){ if (curr_dl->dl_hidden_link==GC_HIDE_POINTER(link)){ curr_dl->dl_hidden_obj=GC_HIDE_POINTER(obj); UNLOCK(); #ifndef DBG_HDRS_ALL GC_free((void*)new_dl); #endif return GC_DUPLICATE; } } } new_dl->dl_hidden_obj=GC_HIDE_POINTER(obj); new_dl->dl_hidden_link=GC_HIDE_POINTER(link); dl_set_next(new_dl,dl_hashtbl->head[index]); GC_dirty(new_dl); dl_hashtbl->head[index]=new_dl; dl_hashtbl->entries++; GC_dirty(dl_hashtbl->head+index); UNLOCK(); return GC_SUCCESS; } GC_API int GC_CALL GC_general_register_disappearing_link(void**link, const void*obj) { if (((word)link&(ALIGNMENT-1))!=0||!NONNULL_ARG_NOT_NULL(link)) ABORT("Bad arg to GC_general_register_disappearing_link"); return GC_register_disappearing_link_inner(&GC_dl_hashtbl,link,obj, "dl"); } #ifdef DBG_HDRS_ALL #define FREE_DL_ENTRY(curr_dl)dl_set_next(curr_dl,NULL) #else #define FREE_DL_ENTRY(curr_dl)GC_free(curr_dl) #endif GC_INLINE struct disappearing_link*GC_unregister_disappearing_link_inner( struct dl_hashtbl_s*dl_hashtbl,void**link) { struct disappearing_link*curr_dl; struct disappearing_link*prev_dl=NULL; size_t index; GC_ASSERT(I_HOLD_LOCK()); if (EXPECT(NULL==dl_hashtbl->head,FALSE))return NULL; index=HASH2(link,dl_hashtbl->log_size); for (curr_dl=dl_hashtbl->head[index];curr_dl; curr_dl=dl_next(curr_dl)){ if (curr_dl->dl_hidden_link==GC_HIDE_POINTER(link)){ if (NULL==prev_dl){ dl_hashtbl->head[index]=dl_next(curr_dl); GC_dirty(dl_hashtbl->head+index); } else { dl_set_next(prev_dl,dl_next(curr_dl)); GC_dirty(prev_dl); } dl_hashtbl->entries--; break; } prev_dl=curr_dl; } return curr_dl; } GC_API int GC_CALL GC_unregister_disappearing_link(void**link) { struct disappearing_link*curr_dl; DCL_LOCK_STATE; if (((word)link&(ALIGNMENT-1))!=0)return(0); LOCK(); curr_dl=GC_unregister_disappearing_link_inner(&GC_dl_hashtbl,link); UNLOCK(); if (NULL==curr_dl)return 0; FREE_DL_ENTRY(curr_dl); return 1; } #ifndef GC_TOGGLE_REFS_NOT_NEEDED typedef union toggle_ref_u GCToggleRef; STATIC GC_toggleref_func GC_toggleref_callback=0; GC_INNER void GC_process_togglerefs(void) { size_t i; size_t new_size=0; GC_bool needs_barrier=FALSE; GC_ASSERT(I_HOLD_LOCK()); for (i=0;i < GC_toggleref_array_size;++i){ GCToggleRef r=GC_toggleref_arr[i]; void*obj=r.strong_ref; if (((word)obj&1)!=0){ obj=GC_REVEAL_POINTER(r.weak_ref); } if (NULL==obj){ continue; } switch (GC_toggleref_callback(obj)){ case GC_TOGGLE_REF_DROP: break; case GC_TOGGLE_REF_STRONG: GC_toggleref_arr[new_size++].strong_ref=obj; needs_barrier=TRUE; break; case GC_TOGGLE_REF_WEAK: GC_toggleref_arr[new_size++].weak_ref=GC_HIDE_POINTER(obj); break; default: ABORT("Bad toggle-ref status returned by callback"); } } if (new_size < GC_toggleref_array_size){ BZERO(&GC_toggleref_arr[new_size], (GC_toggleref_array_size - new_size)*sizeof(GCToggleRef)); GC_toggleref_array_size=new_size; } if (needs_barrier) GC_dirty(GC_toggleref_arr); } STATIC void GC_normal_finalize_mark_proc(ptr_t); static void push_and_mark_object(void*p) { GC_normal_finalize_mark_proc((ptr_t)p); while (!GC_mark_stack_empty()){ MARK_FROM_MARK_STACK(); } GC_set_mark_bit(p); if (GC_mark_state!=MS_NONE){ while (!GC_mark_some(0)){ } } } STATIC void GC_mark_togglerefs(void) { size_t i; if (NULL==GC_toggleref_arr) return; GC_set_mark_bit(GC_toggleref_arr); for (i=0;i < GC_toggleref_array_size;++i){ void*obj=GC_toggleref_arr[i].strong_ref; if (obj!=NULL&&((word)obj&1)==0){ push_and_mark_object(obj); } } } STATIC void GC_clear_togglerefs(void) { size_t i; for (i=0;i < GC_toggleref_array_size;++i){ if ((GC_toggleref_arr[i].weak_ref&1)!=0){ if (!GC_is_marked(GC_REVEAL_POINTER(GC_toggleref_arr[i].weak_ref))){ GC_toggleref_arr[i].weak_ref=0; } else { } } } } GC_API void GC_CALL GC_set_toggleref_func(GC_toggleref_func fn) { DCL_LOCK_STATE; LOCK(); GC_toggleref_callback=fn; UNLOCK(); } GC_API GC_toggleref_func GC_CALL GC_get_toggleref_func(void) { GC_toggleref_func fn; DCL_LOCK_STATE; LOCK(); fn=GC_toggleref_callback; UNLOCK(); return fn; } static GC_bool ensure_toggleref_capacity(size_t capacity_inc) { GC_ASSERT(I_HOLD_LOCK()); if (NULL==GC_toggleref_arr){ GC_toggleref_array_capacity=32; GC_toggleref_arr=(GCToggleRef*)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE( GC_toggleref_array_capacity*sizeof(GCToggleRef), NORMAL); if (NULL==GC_toggleref_arr) return FALSE; } if (GC_toggleref_array_size+capacity_inc >=GC_toggleref_array_capacity){ GCToggleRef*new_array; while (GC_toggleref_array_capacity < GC_toggleref_array_size+capacity_inc){ GC_toggleref_array_capacity*=2; if ((GC_toggleref_array_capacity &((size_t)1<<(sizeof(size_t)*8 - 1)))!=0) return FALSE; } new_array=(GCToggleRef*)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE( GC_toggleref_array_capacity*sizeof(GCToggleRef), NORMAL); if (NULL==new_array) return FALSE; if (EXPECT(GC_toggleref_array_size > 0,TRUE)) BCOPY(GC_toggleref_arr,new_array, GC_toggleref_array_size*sizeof(GCToggleRef)); GC_INTERNAL_FREE(GC_toggleref_arr); GC_toggleref_arr=new_array; } return TRUE; } GC_API int GC_CALL GC_toggleref_add(void*obj,int is_strong_ref) { int res=GC_SUCCESS; DCL_LOCK_STATE; GC_ASSERT(NONNULL_ARG_NOT_NULL(obj)); LOCK(); if (GC_toggleref_callback!=0){ if (!ensure_toggleref_capacity(1)){ res=GC_NO_MEMORY; } else { GC_toggleref_arr[GC_toggleref_array_size].strong_ref= is_strong_ref?obj:(void*)GC_HIDE_POINTER(obj); if (is_strong_ref) GC_dirty(GC_toggleref_arr+GC_toggleref_array_size); GC_toggleref_array_size++; } } UNLOCK(); return res; } #endif STATIC GC_await_finalize_proc GC_object_finalized_proc=0; GC_API void GC_CALL GC_set_await_finalize_proc(GC_await_finalize_proc fn) { DCL_LOCK_STATE; LOCK(); GC_object_finalized_proc=fn; UNLOCK(); } GC_API GC_await_finalize_proc GC_CALL GC_get_await_finalize_proc(void) { GC_await_finalize_proc fn; DCL_LOCK_STATE; LOCK(); fn=GC_object_finalized_proc; UNLOCK(); return fn; } #ifndef GC_LONG_REFS_NOT_NEEDED GC_API int GC_CALL GC_register_long_link(void**link,const void*obj) { if (((word)link&(ALIGNMENT-1))!=0||!NONNULL_ARG_NOT_NULL(link)) ABORT("Bad arg to GC_register_long_link"); return GC_register_disappearing_link_inner(&GC_ll_hashtbl,link,obj, "long dl"); } GC_API int GC_CALL GC_unregister_long_link(void**link) { struct disappearing_link*curr_dl; DCL_LOCK_STATE; if (((word)link&(ALIGNMENT-1))!=0)return(0); LOCK(); curr_dl=GC_unregister_disappearing_link_inner(&GC_ll_hashtbl,link); UNLOCK(); if (NULL==curr_dl)return 0; FREE_DL_ENTRY(curr_dl); return 1; } #endif #ifndef GC_MOVE_DISAPPEARING_LINK_NOT_NEEDED STATIC int GC_move_disappearing_link_inner( struct dl_hashtbl_s*dl_hashtbl, void**link,void**new_link) { struct disappearing_link*curr_dl,*new_dl; struct disappearing_link*prev_dl=NULL; size_t curr_index,new_index; word curr_hidden_link,new_hidden_link; GC_ASSERT(I_HOLD_LOCK()); if (EXPECT(NULL==dl_hashtbl->head,FALSE))return GC_NOT_FOUND; curr_index=HASH2(link,dl_hashtbl->log_size); curr_hidden_link=GC_HIDE_POINTER(link); for (curr_dl=dl_hashtbl->head[curr_index];curr_dl; curr_dl=dl_next(curr_dl)){ if (curr_dl->dl_hidden_link==curr_hidden_link) break; prev_dl=curr_dl; } if (EXPECT(NULL==curr_dl,FALSE)){ return GC_NOT_FOUND; } else if (link==new_link){ return GC_SUCCESS; } new_index=HASH2(new_link,dl_hashtbl->log_size); new_hidden_link=GC_HIDE_POINTER(new_link); for (new_dl=dl_hashtbl->head[new_index];new_dl; new_dl=dl_next(new_dl)){ if (new_dl->dl_hidden_link==new_hidden_link){ return GC_DUPLICATE; } } if (NULL==prev_dl){ dl_hashtbl->head[curr_index]=dl_next(curr_dl); } else { dl_set_next(prev_dl,dl_next(curr_dl)); GC_dirty(prev_dl); } curr_dl->dl_hidden_link=new_hidden_link; dl_set_next(curr_dl,dl_hashtbl->head[new_index]); dl_hashtbl->head[new_index]=curr_dl; GC_dirty(curr_dl); GC_dirty(dl_hashtbl->head); return GC_SUCCESS; } GC_API int GC_CALL GC_move_disappearing_link(void**link,void**new_link) { int result; DCL_LOCK_STATE; if (((word)new_link&(ALIGNMENT-1))!=0 ||!NONNULL_ARG_NOT_NULL(new_link)) ABORT("Bad new_link arg to GC_move_disappearing_link"); if (((word)link&(ALIGNMENT-1))!=0) return GC_NOT_FOUND; LOCK(); result=GC_move_disappearing_link_inner(&GC_dl_hashtbl,link,new_link); UNLOCK(); return result; } #ifndef GC_LONG_REFS_NOT_NEEDED GC_API int GC_CALL GC_move_long_link(void**link,void**new_link) { int result; DCL_LOCK_STATE; if (((word)new_link&(ALIGNMENT-1))!=0 ||!NONNULL_ARG_NOT_NULL(new_link)) ABORT("Bad new_link arg to GC_move_long_link"); if (((word)link&(ALIGNMENT-1))!=0) return GC_NOT_FOUND; LOCK(); result=GC_move_disappearing_link_inner(&GC_ll_hashtbl,link,new_link); UNLOCK(); return result; } #endif #endif STATIC void GC_normal_finalize_mark_proc(ptr_t p) { #if defined(_MSC_VER)&&defined(I386) hdr*hhdr=HDR(p); #define mark_stack_top GC_mark_stack_top mse*mark_stack_limit=GC_mark_stack+GC_mark_stack_size; word descr=hhdr->hb_descr; if (descr!=0){ mark_stack_top++; if ((word)mark_stack_top>=(word)mark_stack_limit){ mark_stack_top=GC_signal_mark_stack_overflow(mark_stack_top); } mark_stack_top->mse_start=p; mark_stack_top->mse_descr.w=descr; } #undef mark_stack_top #else GC_mark_stack_top=GC_push_obj(p,HDR(p),GC_mark_stack_top, GC_mark_stack+GC_mark_stack_size); #endif } STATIC void GC_ignore_self_finalize_mark_proc(ptr_t p) { hdr*hhdr=HDR(p); word descr=hhdr->hb_descr; ptr_t q; ptr_t scan_limit; ptr_t target_limit=p+hhdr->hb_sz - 1; if ((descr&GC_DS_TAGS)==GC_DS_LENGTH){ scan_limit=p+descr - sizeof(word); } else { scan_limit=target_limit+1 - sizeof(word); } for (q=p;(word)q<=(word)scan_limit;q+=ALIGNMENT){ word r=*(word*)q; if (r < (word)p||r > (word)target_limit){ GC_PUSH_ONE_HEAP(r,q,GC_mark_stack_top); } } } STATIC void GC_null_finalize_mark_proc(ptr_t p GC_ATTR_UNUSED){} STATIC void GC_unreachable_finalize_mark_proc(ptr_t p) { GC_normal_finalize_mark_proc(p); } STATIC void GC_register_finalizer_inner(void*obj, GC_finalization_proc fn,void*cd, GC_finalization_proc*ofn,void**ocd, finalization_mark_proc mp) { struct finalizable_object*curr_fo; size_t index; struct finalizable_object*new_fo=0; hdr*hhdr=NULL; DCL_LOCK_STATE; if (EXPECT(GC_find_leak,FALSE))return; LOCK(); if (EXPECT(NULL==GC_fnlz_roots.fo_head,FALSE) ||EXPECT(GC_fo_entries > ((word)1<=sizeof(struct finalizable_object)); if (curr_fo->fo_hidden_base==GC_HIDE_POINTER(obj)){ if (ocd)*ocd=(void*)(curr_fo->fo_client_data); if (ofn)*ofn=curr_fo->fo_fn; if (prev_fo==0){ GC_fnlz_roots.fo_head[index]=fo_next(curr_fo); } else { fo_set_next(prev_fo,fo_next(curr_fo)); GC_dirty(prev_fo); } if (fn==0){ GC_fo_entries--; #if!defined(THREADS)&&!defined(DBG_HDRS_ALL) GC_free((void*)curr_fo); #endif } else { curr_fo->fo_fn=fn; curr_fo->fo_client_data=(ptr_t)cd; curr_fo->fo_mark_proc=mp; GC_dirty(curr_fo); if (prev_fo==0){ GC_fnlz_roots.fo_head[index]=curr_fo; } else { fo_set_next(prev_fo,curr_fo); GC_dirty(prev_fo); } } if (NULL==prev_fo) GC_dirty(GC_fnlz_roots.fo_head+index); UNLOCK(); #ifndef DBG_HDRS_ALL GC_free((void*)new_fo); #endif return; } prev_fo=curr_fo; curr_fo=fo_next(curr_fo); } if (EXPECT(new_fo!=0,FALSE)){ GC_ASSERT(fn!=0); #ifdef LINT2 if (NULL==hhdr)ABORT("Bad hhdr in GC_register_finalizer_inner"); #endif break; } if (fn==0){ if (ocd)*ocd=0; if (ofn)*ofn=0; UNLOCK(); return; } GET_HDR(obj,hhdr); if (EXPECT(0==hhdr,FALSE)){ if (ocd)*ocd=0; if (ofn)*ofn=0; UNLOCK(); return; } new_fo=(struct finalizable_object*) GC_INTERNAL_MALLOC(sizeof(struct finalizable_object),NORMAL); if (EXPECT(new_fo!=0,TRUE)) break; oom_fn=GC_oom_fn; UNLOCK(); new_fo=(struct finalizable_object*) (*oom_fn)(sizeof(struct finalizable_object)); if (0==new_fo){ return; } LOCK(); } GC_ASSERT(GC_size(new_fo)>=sizeof(struct finalizable_object)); if (ocd)*ocd=0; if (ofn)*ofn=0; new_fo->fo_hidden_base=GC_HIDE_POINTER(obj); new_fo->fo_fn=fn; new_fo->fo_client_data=(ptr_t)cd; new_fo->fo_object_size=hhdr->hb_sz; new_fo->fo_mark_proc=mp; fo_set_next(new_fo,GC_fnlz_roots.fo_head[index]); GC_dirty(new_fo); GC_fo_entries++; GC_fnlz_roots.fo_head[index]=new_fo; GC_dirty(GC_fnlz_roots.fo_head+index); UNLOCK(); } GC_API void GC_CALL GC_register_finalizer(void*obj, GC_finalization_proc fn,void*cd, GC_finalization_proc*ofn,void**ocd) { GC_register_finalizer_inner(obj,fn,cd,ofn, ocd,GC_normal_finalize_mark_proc); } GC_API void GC_CALL GC_register_finalizer_ignore_self(void*obj, GC_finalization_proc fn,void*cd, GC_finalization_proc*ofn,void**ocd) { GC_register_finalizer_inner(obj,fn,cd,ofn, ocd,GC_ignore_self_finalize_mark_proc); } GC_API void GC_CALL GC_register_finalizer_no_order(void*obj, GC_finalization_proc fn,void*cd, GC_finalization_proc*ofn,void**ocd) { GC_register_finalizer_inner(obj,fn,cd,ofn, ocd,GC_null_finalize_mark_proc); } static GC_bool need_unreachable_finalization=FALSE; GC_API void GC_CALL GC_register_finalizer_unreachable(void*obj, GC_finalization_proc fn,void*cd, GC_finalization_proc*ofn,void**ocd) { need_unreachable_finalization=TRUE; GC_ASSERT(GC_java_finalization); GC_register_finalizer_inner(obj,fn,cd,ofn, ocd,GC_unreachable_finalize_mark_proc); } #ifndef NO_DEBUGGING STATIC void GC_dump_finalization_links( const struct dl_hashtbl_s*dl_hashtbl) { size_t dl_size=(size_t)1<log_size; size_t i; if (NULL==dl_hashtbl->head)return; for (i=0;i < dl_size;i++){ struct disappearing_link*curr_dl; for (curr_dl=dl_hashtbl->head[i];curr_dl!=0; curr_dl=dl_next(curr_dl)){ ptr_t real_ptr=(ptr_t)GC_REVEAL_POINTER(curr_dl->dl_hidden_obj); ptr_t real_link=(ptr_t)GC_REVEAL_POINTER(curr_dl->dl_hidden_link); GC_printf("Object:%p,link:%p\n", (void*)real_ptr,(void*)real_link); } } } GC_API void GC_CALL GC_dump_finalization(void) { struct finalizable_object*curr_fo; size_t i; size_t fo_size=GC_fnlz_roots.fo_head==NULL?0: (size_t)1<fo_hidden_base); GC_printf("Finalizable object:%p\n",(void*)real_ptr); } } } #endif #ifndef SMALL_CONFIG STATIC word GC_old_dl_entries=0; #ifndef GC_LONG_REFS_NOT_NEEDED STATIC word GC_old_ll_entries=0; #endif #endif #ifndef THREADS STATIC int GC_finalizer_nested=0; STATIC unsigned GC_finalizer_skipped=0; STATIC unsigned char*GC_check_finalizer_nested(void) { unsigned nesting_level=*(unsigned char*)&GC_finalizer_nested; if (nesting_level){ if (++GC_finalizer_skipped < (1U<log_size; GC_bool needs_barrier=FALSE; GC_ASSERT(I_HOLD_LOCK()); if (NULL==dl_hashtbl->head)return; for (i=0;i < dl_size;i++){ struct disappearing_link*curr_dl,*next_dl; struct disappearing_link*prev_dl=NULL; for (curr_dl=dl_hashtbl->head[i];curr_dl!=NULL;curr_dl=next_dl){ next_dl=dl_next(curr_dl); if (is_remove_dangling){ ptr_t real_link=(ptr_t)GC_base(GC_REVEAL_POINTER( curr_dl->dl_hidden_link)); if (NULL==real_link||EXPECT(GC_is_marked(real_link),TRUE)){ prev_dl=curr_dl; continue; } } else { if (EXPECT(GC_is_marked((ptr_t)GC_REVEAL_POINTER( curr_dl->dl_hidden_obj)),TRUE)){ prev_dl=curr_dl; continue; } *(ptr_t*)GC_REVEAL_POINTER(curr_dl->dl_hidden_link)=NULL; } if (NULL==prev_dl){ dl_hashtbl->head[i]=next_dl; needs_barrier=TRUE; } else { dl_set_next(prev_dl,next_dl); GC_dirty(prev_dl); } GC_clear_mark_bit(curr_dl); dl_hashtbl->entries--; } } if (needs_barrier) GC_dirty(dl_hashtbl->head); } GC_INNER void GC_finalize(void) { struct finalizable_object*curr_fo,*prev_fo,*next_fo; ptr_t real_ptr; size_t i; size_t fo_size=GC_fnlz_roots.fo_head==NULL?0: (size_t)1<=sizeof(struct finalizable_object)); real_ptr=(ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base); if (!GC_is_marked(real_ptr)){ GC_MARKED_FOR_FINALIZATION(real_ptr); GC_MARK_FO(real_ptr,curr_fo->fo_mark_proc); if (GC_is_marked(real_ptr)){ WARN("Finalization cycle involving %p\n",real_ptr); } } } } GC_bytes_finalized=0; for (i=0;i < fo_size;i++){ curr_fo=GC_fnlz_roots.fo_head[i]; prev_fo=0; while (curr_fo!=0){ real_ptr=(ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base); if (!GC_is_marked(real_ptr)){ if (!GC_java_finalization){ GC_set_mark_bit(real_ptr); } next_fo=fo_next(curr_fo); if (NULL==prev_fo){ GC_fnlz_roots.fo_head[i]=next_fo; if (GC_object_finalized_proc){ GC_dirty(GC_fnlz_roots.fo_head+i); } else { needs_barrier=TRUE; } } else { fo_set_next(prev_fo,next_fo); GC_dirty(prev_fo); } GC_fo_entries--; if (GC_object_finalized_proc) GC_object_finalized_proc(real_ptr); fo_set_next(curr_fo,GC_fnlz_roots.finalize_now); GC_dirty(curr_fo); SET_FINALIZE_NOW(curr_fo); curr_fo->fo_hidden_base= (word)GC_REVEAL_POINTER(curr_fo->fo_hidden_base); GC_bytes_finalized+= curr_fo->fo_object_size +sizeof(struct finalizable_object); GC_ASSERT(GC_is_marked(GC_base(curr_fo))); curr_fo=next_fo; } else { prev_fo=curr_fo; curr_fo=fo_next(curr_fo); } } } if (GC_java_finalization){ for (curr_fo=GC_fnlz_roots.finalize_now; curr_fo!=NULL;curr_fo=fo_next(curr_fo)){ real_ptr=(ptr_t)curr_fo->fo_hidden_base; if (!GC_is_marked(real_ptr)){ if (curr_fo->fo_mark_proc==GC_null_finalize_mark_proc){ GC_MARK_FO(real_ptr,GC_normal_finalize_mark_proc); } if (curr_fo->fo_mark_proc!=GC_unreachable_finalize_mark_proc){ GC_set_mark_bit(real_ptr); } } } if (need_unreachable_finalization){ curr_fo=GC_fnlz_roots.finalize_now; GC_ASSERT(NULL==curr_fo||GC_fnlz_roots.fo_head!=NULL); prev_fo=NULL; while (curr_fo!=NULL){ next_fo=fo_next(curr_fo); if (curr_fo->fo_mark_proc==GC_unreachable_finalize_mark_proc){ real_ptr=(ptr_t)curr_fo->fo_hidden_base; if (!GC_is_marked(real_ptr)){ GC_set_mark_bit(real_ptr); } else { if (NULL==prev_fo){ SET_FINALIZE_NOW(next_fo); } else { fo_set_next(prev_fo,next_fo); GC_dirty(prev_fo); } curr_fo->fo_hidden_base= GC_HIDE_POINTER(curr_fo->fo_hidden_base); GC_bytes_finalized-= curr_fo->fo_object_size+sizeof(struct finalizable_object); i=HASH2(real_ptr,GC_log_fo_table_size); fo_set_next(curr_fo,GC_fnlz_roots.fo_head[i]); GC_dirty(curr_fo); GC_fo_entries++; GC_fnlz_roots.fo_head[i]=curr_fo; curr_fo=prev_fo; needs_barrier=TRUE; } } prev_fo=curr_fo; curr_fo=next_fo; } } } if (needs_barrier) GC_dirty(GC_fnlz_roots.fo_head); GC_make_disappearing_links_disappear(&GC_dl_hashtbl,TRUE); #ifndef GC_TOGGLE_REFS_NOT_NEEDED GC_clear_togglerefs(); #endif #ifndef GC_LONG_REFS_NOT_NEEDED GC_make_disappearing_links_disappear(&GC_ll_hashtbl,FALSE); GC_make_disappearing_links_disappear(&GC_ll_hashtbl,TRUE); #endif if (GC_fail_count){ #ifdef THREADS GC_reset_finalizer_nested(); #else GC_finalizer_nested=0; #endif } } #ifndef JAVA_FINALIZATION_NOT_NEEDED STATIC void GC_enqueue_all_finalizers(void) { struct finalizable_object*next_fo; size_t i; size_t fo_size=GC_fnlz_roots.fo_head==NULL?0: (size_t)1<fo_hidden_base); GC_MARK_FO(real_ptr,GC_normal_finalize_mark_proc); GC_set_mark_bit(real_ptr); next_fo=fo_next(curr_fo); fo_set_next(curr_fo,GC_fnlz_roots.finalize_now); GC_dirty(curr_fo); SET_FINALIZE_NOW(curr_fo); curr_fo->fo_hidden_base= (word)GC_REVEAL_POINTER(curr_fo->fo_hidden_base); GC_bytes_finalized+= curr_fo->fo_object_size+sizeof(struct finalizable_object); curr_fo=next_fo; } } GC_fo_entries=0; } GC_API void GC_CALL GC_finalize_all(void) { DCL_LOCK_STATE; LOCK(); while (GC_fo_entries > 0){ GC_enqueue_all_finalizers(); UNLOCK(); GC_invoke_finalizers(); LOCK(); } UNLOCK(); } #endif GC_API int GC_CALL GC_should_invoke_finalizers(void) { #ifdef AO_HAVE_load return AO_load((volatile AO_t*)&GC_fnlz_roots.finalize_now)!=0; #else return GC_fnlz_roots.finalize_now!=NULL; #endif } GC_API int GC_CALL GC_invoke_finalizers(void) { int count=0; word bytes_freed_before=0; DCL_LOCK_STATE; while (GC_should_invoke_finalizers()){ struct finalizable_object*curr_fo; #ifdef THREADS LOCK(); #endif if (count==0){ bytes_freed_before=GC_bytes_freed; } curr_fo=GC_fnlz_roots.finalize_now; #ifdef THREADS if (curr_fo!=NULL) SET_FINALIZE_NOW(fo_next(curr_fo)); UNLOCK(); if (curr_fo==0)break; #else GC_fnlz_roots.finalize_now=fo_next(curr_fo); #endif fo_set_next(curr_fo,0); (*(curr_fo->fo_fn))((ptr_t)(curr_fo->fo_hidden_base), curr_fo->fo_client_data); curr_fo->fo_client_data=0; ++count; } if (count!=0 #if defined(THREADS)&&!defined(THREAD_SANITIZER) &&bytes_freed_before!=GC_bytes_freed #endif ){ LOCK(); GC_finalizer_bytes_freed+=(GC_bytes_freed - bytes_freed_before); UNLOCK(); } return count; } static word last_finalizer_notification=0; GC_INNER void GC_notify_or_invoke_finalizers(void) { GC_finalizer_notifier_proc notifier_fn=0; #if defined(KEEP_BACK_PTRS)||defined(MAKE_BACK_GRAPH) static word last_back_trace_gc_no=1; #endif DCL_LOCK_STATE; #if defined(THREADS)&&!defined(KEEP_BACK_PTRS)&&!defined(MAKE_BACK_GRAPH) if (!GC_should_invoke_finalizers()) return; #endif LOCK(); #if defined(KEEP_BACK_PTRS)||defined(MAKE_BACK_GRAPH) if (GC_gc_no > last_back_trace_gc_no){ #ifdef KEEP_BACK_PTRS long i; last_back_trace_gc_no=GC_WORD_MAX; for (i=0;i < GC_backtraces;++i){ UNLOCK(); GC_generate_random_backtrace_no_gc(); LOCK(); } last_back_trace_gc_no=GC_gc_no; #endif #ifdef MAKE_BACK_GRAPH if (GC_print_back_height){ GC_print_back_graph_stats(); } #endif } #endif if (NULL==GC_fnlz_roots.finalize_now){ UNLOCK(); return; } if (!GC_finalize_on_demand){ unsigned char*pnested=GC_check_finalizer_nested(); UNLOCK(); if (pnested!=NULL){ (void)GC_invoke_finalizers(); *pnested=0; #ifndef THREADS GC_ASSERT(NULL==GC_fnlz_roots.finalize_now); #endif } return; } if (last_finalizer_notification!=GC_gc_no){ notifier_fn=GC_finalizer_notifier; last_finalizer_notification=GC_gc_no; } UNLOCK(); if (notifier_fn!=0) (*notifier_fn)(); } #ifndef SMALL_CONFIG #ifndef GC_LONG_REFS_NOT_NEEDED #define IF_LONG_REFS_PRESENT_ELSE(x,y)(x) #else #define IF_LONG_REFS_PRESENT_ELSE(x,y)(y) #endif GC_INNER void GC_print_finalization_stats(void) { struct finalizable_object*fo; unsigned long ready=0; GC_log_printf("%lu finalization entries;" " %lu/%lu short/long disappearing links alive\n", (unsigned long)GC_fo_entries, (unsigned long)GC_dl_hashtbl.entries, (unsigned long)IF_LONG_REFS_PRESENT_ELSE( GC_ll_hashtbl.entries,0)); for (fo=GC_fnlz_roots.finalize_now;fo!=NULL;fo=fo_next(fo)) ++ready; GC_log_printf("%lu finalization-ready objects;" " %ld/%ld short/long links cleared\n", ready, (long)GC_old_dl_entries - (long)GC_dl_hashtbl.entries, (long)IF_LONG_REFS_PRESENT_ELSE( GC_old_ll_entries - GC_ll_hashtbl.entries,0)); } #endif #endif #ifdef ENABLE_DISCLAIM #ifndef GC_DISCLAIM_H #define GC_DISCLAIM_H #ifdef __cplusplus extern "C" { #endif GC_API void GC_CALL GC_init_finalized_malloc(void); typedef int (GC_CALLBACK*GC_disclaim_proc)(void*); GC_API void GC_CALL GC_register_disclaim_proc(int, GC_disclaim_proc, int)GC_ATTR_NONNULL(2); struct GC_finalizer_closure { GC_finalization_proc proc; void*cd; }; GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_finalized_malloc(size_t, const struct GC_finalizer_closure*)GC_ATTR_NONNULL(2); #ifdef __cplusplus } #endif #endif #if defined(KEEP_BACK_PTRS)||defined(MAKE_BACK_GRAPH) #define FINALIZER_CLOSURE_FLAG 0x2 #else #define FINALIZER_CLOSURE_FLAG 0x1 #endif STATIC int GC_CALLBACK GC_finalized_disclaim(void*obj) { word fc_word=*(word*)obj; if ((fc_word&FINALIZER_CLOSURE_FLAG)!=0){ const struct GC_finalizer_closure*fc =(struct GC_finalizer_closure*)(fc_word &~(word)FINALIZER_CLOSURE_FLAG); GC_ASSERT(!GC_find_leak); (*fc->proc)((word*)obj+1,fc->cd); } return 0; } GC_API void GC_CALL GC_init_finalized_malloc(void) { DCL_LOCK_STATE; GC_init(); LOCK(); if (GC_finalized_kind!=0){ UNLOCK(); return; } GC_register_displacement_inner(sizeof(word)); GC_register_displacement_inner(FINALIZER_CLOSURE_FLAG); GC_register_displacement_inner(sizeof(oh)+FINALIZER_CLOSURE_FLAG); GC_finalized_kind=GC_new_kind_inner(GC_new_free_list_inner(), GC_DS_LENGTH,TRUE,TRUE); GC_ASSERT(GC_finalized_kind!=0); GC_register_disclaim_proc(GC_finalized_kind,GC_finalized_disclaim,TRUE); UNLOCK(); } GC_API void GC_CALL GC_register_disclaim_proc(int kind,GC_disclaim_proc proc, int mark_unconditionally) { GC_ASSERT((unsigned)kind < MAXOBJKINDS); GC_ASSERT(NONNULL_ARG_NOT_NULL(proc)); if (!EXPECT(GC_find_leak,FALSE)){ GC_obj_kinds[kind].ok_disclaim_proc=proc; GC_obj_kinds[kind].ok_mark_unconditionally= (GC_bool)mark_unconditionally; } } GC_API GC_ATTR_MALLOC void*GC_CALL GC_finalized_malloc(size_t lb, const struct GC_finalizer_closure*fclos) { word*op; GC_ASSERT(GC_finalized_kind!=0); GC_ASSERT(NONNULL_ARG_NOT_NULL(fclos)); GC_ASSERT(((word)fclos&FINALIZER_CLOSURE_FLAG)==0); op=(word*)GC_malloc_kind(SIZET_SAT_ADD(lb,sizeof(word)), GC_finalized_kind); if (EXPECT(NULL==op,FALSE)) return NULL; *op=(word)fclos|FINALIZER_CLOSURE_FLAG; GC_dirty(op); REACHABLE_AFTER_DIRTY(fclos); return op+1; } #endif #include #include STATIC GC_bool GC_alloc_reclaim_list(struct obj_kind*kind) { struct hblk**result=(struct hblk**) GC_scratch_alloc((MAXOBJGRANULES+1)*sizeof(struct hblk*)); if (result==0)return(FALSE); BZERO(result,(MAXOBJGRANULES+1)*sizeof(struct hblk*)); kind->ok_reclaim_list=result; return(TRUE); } GC_INNER ptr_t GC_alloc_large(size_t lb,int k,unsigned flags) { struct hblk*h; word n_blocks; ptr_t result; GC_bool retry=FALSE; GC_ASSERT(I_HOLD_LOCK()); lb=ROUNDUP_GRANULE_SIZE(lb); n_blocks=OBJ_SZ_TO_BLOCKS_CHECKED(lb); if (!EXPECT(GC_is_initialized,TRUE)){ DCL_LOCK_STATE; UNLOCK(); GC_init(); LOCK(); } if (GC_incremental&&!GC_dont_gc){ ENTER_GC(); GC_collect_a_little_inner((int)n_blocks); EXIT_GC(); } h=GC_allochblk(lb,k,flags); #ifdef USE_MUNMAP if (0==h){ GC_merge_unmapped(); h=GC_allochblk(lb,k,flags); } #endif while (0==h&&GC_collect_or_expand(n_blocks,flags!=0,retry)){ h=GC_allochblk(lb,k,flags); retry=TRUE; } if (h==0){ result=0; } else { size_t total_bytes=n_blocks*HBLKSIZE; if (n_blocks > 1){ GC_large_allocd_bytes+=total_bytes; if (GC_large_allocd_bytes > GC_max_large_allocd_bytes) GC_max_large_allocd_bytes=GC_large_allocd_bytes; } result=h->hb_body; } return result; } STATIC ptr_t GC_alloc_large_and_clear(size_t lb,int k,unsigned flags) { ptr_t result; GC_ASSERT(I_HOLD_LOCK()); result=GC_alloc_large(lb,k,flags); if (result!=NULL &&(GC_debugging_started||GC_obj_kinds[k].ok_init)){ word n_blocks=OBJ_SZ_TO_BLOCKS(lb); BZERO(result,n_blocks*HBLKSIZE); } return result; } STATIC void GC_extend_size_map(size_t i) { size_t orig_granule_sz=ROUNDED_UP_GRANULES(i); size_t granule_sz; size_t byte_sz=GRANULES_TO_BYTES(orig_granule_sz); size_t smaller_than_i=byte_sz - (byte_sz>>3); size_t low_limit; size_t number_of_objs; GC_ASSERT(I_HOLD_LOCK()); GC_ASSERT(0==GC_size_map[i]); if (0==GC_size_map[smaller_than_i]){ low_limit=byte_sz - (byte_sz>>2); granule_sz=orig_granule_sz; while (GC_size_map[low_limit]!=0) low_limit++; } else { low_limit=smaller_than_i+1; while (GC_size_map[low_limit]!=0) low_limit++; granule_sz=ROUNDED_UP_GRANULES(low_limit); granule_sz+=granule_sz>>3; if (granule_sz < orig_granule_sz) granule_sz=orig_granule_sz; } granule_sz=(granule_sz+1)&~1; if (granule_sz > MAXOBJGRANULES) granule_sz=MAXOBJGRANULES; number_of_objs=HBLK_GRANULES/granule_sz; GC_ASSERT(number_of_objs!=0); granule_sz=(HBLK_GRANULES/number_of_objs)&~1; byte_sz=GRANULES_TO_BYTES(granule_sz)- EXTRA_BYTES; for (;low_limit<=byte_sz;low_limit++) GC_size_map[low_limit]=granule_sz; } GC_INNER void*GC_generic_malloc_inner(size_t lb,int k) { void*op; GC_ASSERT(I_HOLD_LOCK()); GC_ASSERT(k < MAXOBJKINDS); if (SMALL_OBJ(lb)){ struct obj_kind*kind=GC_obj_kinds+k; size_t lg=GC_size_map[lb]; void**opp=&(kind->ok_freelist[lg]); op=*opp; if (EXPECT(0==op,FALSE)){ if (lg==0){ if (!EXPECT(GC_is_initialized,TRUE)){ DCL_LOCK_STATE; UNLOCK(); GC_init(); LOCK(); lg=GC_size_map[lb]; } if (0==lg){ GC_extend_size_map(lb); lg=GC_size_map[lb]; GC_ASSERT(lg!=0); } opp=&(kind->ok_freelist[lg]); op=*opp; } if (0==op){ if (0==kind->ok_reclaim_list&& !GC_alloc_reclaim_list(kind)) return NULL; op=GC_allocobj(lg,k); if (0==op) return NULL; } } *opp=obj_link(op); obj_link(op)=0; GC_bytes_allocd+=GRANULES_TO_BYTES((word)lg); } else { op=(ptr_t)GC_alloc_large_and_clear(ADD_SLOP(lb),k,0); if (op!=NULL) GC_bytes_allocd+=lb; } return op; } #if defined(DBG_HDRS_ALL)||defined(GC_GCJ_SUPPORT)||!defined(GC_NO_FINALIZATION) GC_INNER void*GC_generic_malloc_inner_ignore_off_page(size_t lb,int k) { word lb_adjusted; void*op; GC_ASSERT(I_HOLD_LOCK()); if (lb<=HBLKSIZE) return GC_generic_malloc_inner(lb,k); GC_ASSERT(k < MAXOBJKINDS); lb_adjusted=ADD_SLOP(lb); op=GC_alloc_large_and_clear(lb_adjusted,k,IGNORE_OFF_PAGE); if (op!=NULL) GC_bytes_allocd+=lb_adjusted; return op; } #endif #ifdef GC_COLLECT_AT_MALLOC #if defined(CPPCHECK) size_t GC_dbg_collect_at_malloc_min_lb=16*1024; #else size_t GC_dbg_collect_at_malloc_min_lb=(GC_COLLECT_AT_MALLOC); #endif #endif GC_API GC_ATTR_MALLOC void*GC_CALL GC_generic_malloc(size_t lb,int k) { void*result; DCL_LOCK_STATE; GC_ASSERT(k < MAXOBJKINDS); if (EXPECT(GC_have_errors,FALSE)) GC_print_all_errors(); GC_INVOKE_FINALIZERS(); GC_DBG_COLLECT_AT_MALLOC(lb); if (SMALL_OBJ(lb)){ LOCK(); result=GC_generic_malloc_inner(lb,k); UNLOCK(); } else { size_t lg; size_t lb_rounded; word n_blocks; GC_bool init; lg=ROUNDED_UP_GRANULES(lb); lb_rounded=GRANULES_TO_BYTES(lg); n_blocks=OBJ_SZ_TO_BLOCKS(lb_rounded); init=GC_obj_kinds[k].ok_init; LOCK(); result=(ptr_t)GC_alloc_large(lb_rounded,k,0); if (0!=result){ if (GC_debugging_started){ BZERO(result,n_blocks*HBLKSIZE); } else { #ifdef THREADS ((word*)result)[0]=0; ((word*)result)[1]=0; ((word*)result)[GRANULES_TO_WORDS(lg)-1]=0; ((word*)result)[GRANULES_TO_WORDS(lg)-2]=0; #endif } GC_bytes_allocd+=lb_rounded; } UNLOCK(); if (init&&!GC_debugging_started&&0!=result){ BZERO(result,n_blocks*HBLKSIZE); } } if (0==result){ return((*GC_get_oom_fn())(lb)); } else { return(result); } } GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_kind_global(size_t lb,int k) { GC_ASSERT(k < MAXOBJKINDS); if (SMALL_OBJ(lb)){ void*op; void**opp; size_t lg; DCL_LOCK_STATE; GC_DBG_COLLECT_AT_MALLOC(lb); LOCK(); lg=GC_size_map[lb]; opp=&GC_obj_kinds[k].ok_freelist[lg]; op=*opp; if (EXPECT(op!=NULL,TRUE)){ if (k==PTRFREE){ *opp=obj_link(op); } else { GC_ASSERT(0==obj_link(op) ||((word)obj_link(op) <=(word)GC_greatest_plausible_heap_addr &&(word)obj_link(op) >=(word)GC_least_plausible_heap_addr)); *opp=obj_link(op); obj_link(op)=0; } GC_bytes_allocd+=GRANULES_TO_BYTES((word)lg); UNLOCK(); return op; } UNLOCK(); } return GC_clear_stack(GC_generic_malloc(lb,k)); } #if defined(THREADS)&&!defined(THREAD_LOCAL_ALLOC) GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_kind(size_t lb,int k) { return GC_malloc_kind_global(lb,k); } #endif GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_atomic(size_t lb) { return GC_malloc_kind(lb,PTRFREE); } GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc(size_t lb) { return GC_malloc_kind(lb,NORMAL); } GC_API GC_ATTR_MALLOC void*GC_CALL GC_generic_malloc_uncollectable( size_t lb,int k) { void*op; DCL_LOCK_STATE; GC_ASSERT(k < MAXOBJKINDS); if (SMALL_OBJ(lb)){ void**opp; size_t lg; GC_DBG_COLLECT_AT_MALLOC(lb); if (EXTRA_BYTES!=0&&lb!=0)lb--; LOCK(); lg=GC_size_map[lb]; opp=&GC_obj_kinds[k].ok_freelist[lg]; op=*opp; if (EXPECT(op!=NULL,TRUE)){ *opp=obj_link(op); obj_link(op)=0; GC_bytes_allocd+=GRANULES_TO_BYTES((word)lg); GC_non_gc_bytes+=GRANULES_TO_BYTES((word)lg); UNLOCK(); } else { UNLOCK(); op=GC_generic_malloc(lb,k); } GC_ASSERT(0==op||GC_is_marked(op)); } else { op=GC_generic_malloc(lb,k); if (op){ hdr*hhdr=HDR(op); GC_ASSERT(((word)op&(HBLKSIZE - 1))==0); LOCK(); set_mark_bit_from_hdr(hhdr,0); #ifndef THREADS GC_ASSERT(hhdr->hb_n_marks==0); #endif hhdr->hb_n_marks=1; UNLOCK(); } } return op; } GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_uncollectable(size_t lb) { return GC_generic_malloc_uncollectable(lb,UNCOLLECTABLE); } #ifdef GC_ATOMIC_UNCOLLECTABLE GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_atomic_uncollectable(size_t lb) { return GC_generic_malloc_uncollectable(lb,AUNCOLLECTABLE); } #endif #if defined(REDIRECT_MALLOC)&&!defined(REDIRECT_MALLOC_IN_HEADER) #ifndef MSWINCE #include #endif #define GC_debug_malloc_replacement(lb)GC_debug_malloc(lb,GC_DBG_EXTRAS) #if defined(CPPCHECK) #define REDIRECT_MALLOC_F GC_malloc #else #define REDIRECT_MALLOC_F REDIRECT_MALLOC #endif void*malloc(size_t lb) { #if defined(I386)&&defined(GC_SOLARIS_THREADS) if (!EXPECT(GC_is_initialized,TRUE))return sbrk(lb); #endif return (void*)REDIRECT_MALLOC_F(lb); } #if defined(GC_LINUX_THREADS) STATIC ptr_t GC_libpthread_start=0; STATIC ptr_t GC_libpthread_end=0; STATIC ptr_t GC_libld_start=0; STATIC ptr_t GC_libld_end=0; STATIC void GC_init_lib_bounds(void) { IF_CANCEL(int cancel_state;) if (GC_libpthread_start!=0)return; DISABLE_CANCEL(cancel_state); GC_init(); if (!GC_text_mapping("libpthread-", &GC_libpthread_start,&GC_libpthread_end)){ WARN("Failed to find libpthread.so text mapping:Expect crash\n",0); GC_libpthread_start=(ptr_t)1; } if (!GC_text_mapping("ld-",&GC_libld_start,&GC_libld_end)){ WARN("Failed to find ld.so text mapping:Expect crash\n",0); } RESTORE_CANCEL(cancel_state); } #endif void*calloc(size_t n,size_t lb) { if ((lb|n)> GC_SQRT_SIZE_MAX &&lb&&n > GC_SIZE_MAX/lb) return (*GC_get_oom_fn())(GC_SIZE_MAX); #if defined(GC_LINUX_THREADS) { static GC_bool lib_bounds_set=FALSE; ptr_t caller=(ptr_t)__builtin_return_address(0); if (!EXPECT(lib_bounds_set,TRUE)){ GC_init_lib_bounds(); lib_bounds_set=TRUE; } if (((word)caller>=(word)GC_libpthread_start &&(word)caller < (word)GC_libpthread_end) ||((word)caller>=(word)GC_libld_start &&(word)caller < (word)GC_libld_end)) return GC_generic_malloc_uncollectable(n*lb,UNCOLLECTABLE); } #endif return (void*)REDIRECT_MALLOC_F(n*lb); } #ifndef strdup char*strdup(const char*s) { size_t lb=strlen(s)+1; char*result=(char*)REDIRECT_MALLOC_F(lb); if (result==0){ errno=ENOMEM; return 0; } BCOPY(s,result,lb); return result; } #endif #ifndef strndup char*strndup(const char*str,size_t size) { char*copy; size_t len=strlen(str); if (len > size) len=size; copy=(char*)REDIRECT_MALLOC_F(len+1); if (copy==NULL){ errno=ENOMEM; return NULL; } if (EXPECT(len > 0,TRUE)) BCOPY(str,copy,len); copy[len]='\0'; return copy; } #endif #undef GC_debug_malloc_replacement #endif GC_API void GC_CALL GC_free(void*p) { struct hblk*h; hdr*hhdr; size_t sz; size_t ngranules; int knd; struct obj_kind*ok; DCL_LOCK_STATE; if (p){ } else { return; } #ifdef LOG_ALLOCS GC_log_printf("GC_free(%p)after GC #%lu\n", p,(unsigned long)GC_gc_no); #endif h=HBLKPTR(p); hhdr=HDR(h); #if defined(REDIRECT_MALLOC)&&((defined(NEED_CALLINFO)&&defined(GC_HAVE_BUILTIN_BACKTRACE))||defined(GC_SOLARIS_THREADS)||defined(GC_LINUX_THREADS)||defined(MSWIN32)) if (0==hhdr)return; #endif GC_ASSERT(GC_base(p)==p); sz=(size_t)hhdr->hb_sz; ngranules=BYTES_TO_GRANULES(sz); knd=hhdr->hb_obj_kind; ok=&GC_obj_kinds[knd]; if (EXPECT(ngranules<=MAXOBJGRANULES,TRUE)){ void**flh; LOCK(); GC_bytes_freed+=sz; if (IS_UNCOLLECTABLE(knd))GC_non_gc_bytes-=sz; if (ok->ok_init&&EXPECT(sz > sizeof(word),TRUE)){ BZERO((word*)p+1,sz-sizeof(word)); } flh=&(ok->ok_freelist[ngranules]); obj_link(p)=*flh; *flh=(ptr_t)p; UNLOCK(); } else { size_t nblocks=OBJ_SZ_TO_BLOCKS(sz); LOCK(); GC_bytes_freed+=sz; if (IS_UNCOLLECTABLE(knd))GC_non_gc_bytes-=sz; if (nblocks > 1){ GC_large_allocd_bytes-=nblocks*HBLKSIZE; } GC_freehblk(h); UNLOCK(); } } #ifdef THREADS GC_INNER void GC_free_inner(void*p) { struct hblk*h; hdr*hhdr; size_t sz; size_t ngranules; int knd; struct obj_kind*ok; h=HBLKPTR(p); hhdr=HDR(h); knd=hhdr->hb_obj_kind; sz=(size_t)hhdr->hb_sz; ngranules=BYTES_TO_GRANULES(sz); ok=&GC_obj_kinds[knd]; if (ngranules<=MAXOBJGRANULES){ void**flh; GC_bytes_freed+=sz; if (IS_UNCOLLECTABLE(knd))GC_non_gc_bytes-=sz; if (ok->ok_init&&EXPECT(sz > sizeof(word),TRUE)){ BZERO((word*)p+1,sz-sizeof(word)); } flh=&(ok->ok_freelist[ngranules]); obj_link(p)=*flh; *flh=(ptr_t)p; } else { size_t nblocks=OBJ_SZ_TO_BLOCKS(sz); GC_bytes_freed+=sz; if (IS_UNCOLLECTABLE(knd))GC_non_gc_bytes-=sz; if (nblocks > 1){ GC_large_allocd_bytes-=nblocks*HBLKSIZE; } GC_freehblk(h); } } #endif #if defined(REDIRECT_MALLOC)&&!defined(REDIRECT_FREE) #define REDIRECT_FREE GC_free #endif #if defined(REDIRECT_FREE)&&!defined(REDIRECT_MALLOC_IN_HEADER) #if defined(CPPCHECK) #define REDIRECT_FREE_F GC_free #else #define REDIRECT_FREE_F REDIRECT_FREE #endif void free(void*p) { #ifndef IGNORE_FREE #if defined(GC_LINUX_THREADS)&&!defined(USE_PROC_FOR_LIBRARIES) ptr_t caller=(ptr_t)__builtin_return_address(0); if (((word)caller>=(word)GC_libpthread_start &&(word)caller < (word)GC_libpthread_end) ||((word)caller>=(word)GC_libld_start &&(word)caller < (word)GC_libld_end)){ GC_free(p); return; } #endif REDIRECT_FREE_F(p); #endif } #endif #include #include #ifndef MSWINCE #include #endif #ifndef GC_ALLOC_PTRS_H #define GC_ALLOC_PTRS_H #ifdef __cplusplus extern "C" { #endif GC_API void**const GC_objfreelist_ptr; GC_API void**const GC_aobjfreelist_ptr; GC_API void**const GC_uobjfreelist_ptr; #ifdef GC_ATOMIC_UNCOLLECTABLE GC_API void**const GC_auobjfreelist_ptr; #endif GC_API void GC_CALL GC_incr_bytes_allocd(size_t bytes); GC_API void GC_CALL GC_incr_bytes_freed(size_t bytes); #ifdef __cplusplus } #endif #endif void**const GC_objfreelist_ptr=GC_objfreelist; void**const GC_aobjfreelist_ptr=GC_aobjfreelist; void**const GC_uobjfreelist_ptr=GC_uobjfreelist; #ifdef GC_ATOMIC_UNCOLLECTABLE void**const GC_auobjfreelist_ptr=GC_auobjfreelist; #endif GC_API int GC_CALL GC_get_kind_and_size(const void*p,size_t*psize) { hdr*hhdr=HDR(p); if (psize!=NULL){ *psize=(size_t)hhdr->hb_sz; } return hhdr->hb_obj_kind; } GC_API GC_ATTR_MALLOC void*GC_CALL GC_generic_or_special_malloc(size_t lb, int knd) { switch(knd){ case PTRFREE: case NORMAL: return GC_malloc_kind(lb,knd); case UNCOLLECTABLE: #ifdef GC_ATOMIC_UNCOLLECTABLE case AUNCOLLECTABLE: #endif return GC_generic_malloc_uncollectable(lb,knd); default: return GC_generic_malloc(lb,knd); } } GC_API void*GC_CALL GC_realloc(void*p,size_t lb) { struct hblk*h; hdr*hhdr; void*result; size_t sz; size_t orig_sz; int obj_kind; if (p==0)return(GC_malloc(lb)); if (0==lb){ #ifndef IGNORE_FREE GC_free(p); #endif return NULL; } h=HBLKPTR(p); hhdr=HDR(h); sz=(size_t)hhdr->hb_sz; obj_kind=hhdr->hb_obj_kind; orig_sz=sz; if (sz > MAXOBJBYTES){ word descr=GC_obj_kinds[obj_kind].ok_descriptor; sz=(sz+HBLKSIZE-1)&~HBLKMASK; if (GC_obj_kinds[obj_kind].ok_relocate_descr) descr+=sz; #ifdef AO_HAVE_store GC_STATIC_ASSERT(sizeof(hhdr->hb_sz)==sizeof(AO_t)); AO_store((volatile AO_t*)&hhdr->hb_sz,(AO_t)sz); AO_store((volatile AO_t*)&hhdr->hb_descr,(AO_t)descr); #else { DCL_LOCK_STATE; LOCK(); hhdr->hb_sz=sz; hhdr->hb_descr=descr; UNLOCK(); } #endif #ifdef MARK_BIT_PER_OBJ GC_ASSERT(hhdr->hb_inv_sz==LARGE_INV_SZ); #endif #ifdef MARK_BIT_PER_GRANULE GC_ASSERT((hhdr->hb_flags&LARGE_BLOCK)!=0 &&hhdr->hb_map[ANY_INDEX]==1); #endif if (IS_UNCOLLECTABLE(obj_kind))GC_non_gc_bytes+=(sz - orig_sz); } if (ADD_SLOP(lb)<=sz){ if (lb>=(sz>>1)){ if (orig_sz > lb){ BZERO(((ptr_t)p)+lb,orig_sz - lb); } return(p); } sz=lb; } result=GC_generic_or_special_malloc((word)lb,obj_kind); if (result!=NULL){ BCOPY(p,result,sz); #ifndef IGNORE_FREE GC_free(p); #endif } return result; } #if defined(REDIRECT_MALLOC)&&!defined(REDIRECT_REALLOC) #define REDIRECT_REALLOC GC_realloc #endif #ifdef REDIRECT_REALLOC #define GC_debug_realloc_replacement(p,lb)GC_debug_realloc(p,lb,GC_DBG_EXTRAS) #if!defined(REDIRECT_MALLOC_IN_HEADER) void*realloc(void*p,size_t lb) { return(REDIRECT_REALLOC(p,lb)); } #endif #undef GC_debug_realloc_replacement #endif GC_API GC_ATTR_MALLOC void*GC_CALL GC_generic_malloc_ignore_off_page(size_t lb,int k) { void*result; size_t lg; size_t lb_rounded; word n_blocks; GC_bool init; DCL_LOCK_STATE; if (SMALL_OBJ(lb)) return GC_generic_malloc(lb,k); GC_ASSERT(k < MAXOBJKINDS); lg=ROUNDED_UP_GRANULES(lb); lb_rounded=GRANULES_TO_BYTES(lg); n_blocks=OBJ_SZ_TO_BLOCKS(lb_rounded); init=GC_obj_kinds[k].ok_init; if (EXPECT(GC_have_errors,FALSE)) GC_print_all_errors(); GC_INVOKE_FINALIZERS(); GC_DBG_COLLECT_AT_MALLOC(lb); LOCK(); result=(ptr_t)GC_alloc_large(ADD_SLOP(lb),k,IGNORE_OFF_PAGE); if (NULL==result){ GC_oom_func oom_fn=GC_oom_fn; UNLOCK(); return (*oom_fn)(lb); } if (GC_debugging_started){ BZERO(result,n_blocks*HBLKSIZE); } else { #ifdef THREADS ((word*)result)[0]=0; ((word*)result)[1]=0; ((word*)result)[GRANULES_TO_WORDS(lg)-1]=0; ((word*)result)[GRANULES_TO_WORDS(lg)-2]=0; #endif } GC_bytes_allocd+=lb_rounded; UNLOCK(); if (init&&!GC_debugging_started){ BZERO(result,n_blocks*HBLKSIZE); } return(result); } GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_ignore_off_page(size_t lb) { return GC_generic_malloc_ignore_off_page(lb,NORMAL); } GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_atomic_ignore_off_page(size_t lb) { return GC_generic_malloc_ignore_off_page(lb,PTRFREE); } GC_API void GC_CALL GC_incr_bytes_allocd(size_t n) { GC_bytes_allocd+=n; } GC_API void GC_CALL GC_incr_bytes_freed(size_t n) { GC_bytes_freed+=n; } GC_API size_t GC_CALL GC_get_expl_freed_bytes_since_gc(void) { return (size_t)GC_bytes_freed; } #ifdef PARALLEL_MARK STATIC volatile AO_t GC_bytes_allocd_tmp=0; #endif GC_API void GC_CALL GC_generic_malloc_many(size_t lb,int k,void**result) { void*op; void*p; void**opp; size_t lw; size_t lg; signed_word my_bytes_allocd=0; struct obj_kind*ok=&(GC_obj_kinds[k]); struct hblk**rlh; DCL_LOCK_STATE; GC_ASSERT(lb!=0&&(lb&(GRANULE_BYTES-1))==0); if (!SMALL_OBJ(lb)||GC_manual_vdb){ op=GC_generic_malloc(lb,k); if (EXPECT(0!=op,TRUE)) obj_link(op)=0; *result=op; #ifndef GC_DISABLE_INCREMENTAL if (GC_manual_vdb&&GC_is_heap_ptr(result)){ GC_dirty_inner(result); REACHABLE_AFTER_DIRTY(op); } #endif return; } GC_ASSERT(k < MAXOBJKINDS); lw=BYTES_TO_WORDS(lb); lg=BYTES_TO_GRANULES(lb); if (EXPECT(GC_have_errors,FALSE)) GC_print_all_errors(); GC_INVOKE_FINALIZERS(); GC_DBG_COLLECT_AT_MALLOC(lb); if (!EXPECT(GC_is_initialized,TRUE))GC_init(); LOCK(); if (GC_incremental&&!GC_dont_gc){ ENTER_GC(); GC_collect_a_little_inner(1); EXIT_GC(); } rlh=ok->ok_reclaim_list; if (rlh!=NULL){ struct hblk*hbp; hdr*hhdr; for (rlh+=lg;(hbp=*rlh)!=NULL;){ hhdr=HDR(hbp); *rlh=hhdr->hb_next; GC_ASSERT(hhdr->hb_sz==lb); hhdr->hb_last_reclaimed=(unsigned short)GC_gc_no; #ifdef PARALLEL_MARK if (GC_parallel){ signed_word my_bytes_allocd_tmp= (signed_word)AO_load(&GC_bytes_allocd_tmp); GC_ASSERT(my_bytes_allocd_tmp>=0); if (my_bytes_allocd_tmp!=0){ (void)AO_fetch_and_add(&GC_bytes_allocd_tmp, (AO_t)(-my_bytes_allocd_tmp)); GC_bytes_allocd+=my_bytes_allocd_tmp; } GC_acquire_mark_lock(); ++GC_fl_builder_count; UNLOCK(); GC_release_mark_lock(); } #endif op=GC_reclaim_generic(hbp,hhdr,lb, ok->ok_init,0,&my_bytes_allocd); if (op!=0){ #ifdef PARALLEL_MARK if (GC_parallel){ *result=op; (void)AO_fetch_and_add(&GC_bytes_allocd_tmp, (AO_t)my_bytes_allocd); GC_acquire_mark_lock(); --GC_fl_builder_count; if (GC_fl_builder_count==0)GC_notify_all_builder(); #ifdef THREAD_SANITIZER GC_release_mark_lock(); LOCK(); GC_bytes_found+=my_bytes_allocd; UNLOCK(); #else GC_bytes_found+=my_bytes_allocd; GC_release_mark_lock(); #endif (void)GC_clear_stack(0); return; } #endif GC_bytes_found+=my_bytes_allocd; GC_bytes_allocd+=my_bytes_allocd; goto out; } #ifdef PARALLEL_MARK if (GC_parallel){ GC_acquire_mark_lock(); --GC_fl_builder_count; if (GC_fl_builder_count==0)GC_notify_all_builder(); GC_release_mark_lock(); LOCK(); } #endif } } opp=&(GC_obj_kinds[k].ok_freelist[lg]); if ( (op=*opp)!=0){ *opp=0; my_bytes_allocd=0; for (p=op;p!=0;p=obj_link(p)){ my_bytes_allocd+=lb; if ((word)my_bytes_allocd>=HBLKSIZE){ *opp=obj_link(p); obj_link(p)=0; break; } } GC_bytes_allocd+=my_bytes_allocd; goto out; } { struct hblk*h=GC_allochblk(lb,k,0); if (h){ if (IS_UNCOLLECTABLE(k))GC_set_hdr_marks(HDR(h)); GC_bytes_allocd+=HBLKSIZE - HBLKSIZE % lb; #ifdef PARALLEL_MARK if (GC_parallel){ GC_acquire_mark_lock(); ++GC_fl_builder_count; UNLOCK(); GC_release_mark_lock(); op=GC_build_fl(h,lw, (ok->ok_init||GC_debugging_started),0); *result=op; GC_acquire_mark_lock(); --GC_fl_builder_count; if (GC_fl_builder_count==0)GC_notify_all_builder(); GC_release_mark_lock(); (void)GC_clear_stack(0); return; } #endif op=GC_build_fl(h,lw,(ok->ok_init||GC_debugging_started),0); goto out; } } op=GC_generic_malloc_inner(lb,k); if (0!=op)obj_link(op)=0; out: *result=op; UNLOCK(); (void)GC_clear_stack(0); } GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_many(size_t lb) { void*result; lb=SIZET_SAT_ADD(lb,EXTRA_BYTES+GRANULE_BYTES - 1) &~(GRANULE_BYTES - 1); GC_generic_malloc_many(lb,NORMAL,&result); return result; } #include GC_API GC_ATTR_MALLOC void*GC_CALL GC_memalign(size_t align,size_t lb) { size_t new_lb; size_t offset; ptr_t result; if (align<=GRANULE_BYTES)return GC_malloc(lb); if (align>=HBLKSIZE/2||lb>=HBLKSIZE/2){ if (align > HBLKSIZE){ return (*GC_get_oom_fn())(LONG_MAX-1024); } return GC_malloc(lb<=HBLKSIZE?HBLKSIZE:lb); } new_lb=SIZET_SAT_ADD(lb,align - 1); result=(ptr_t)GC_malloc(new_lb); offset=(word)result % align; if (offset!=0){ offset=align - offset; if (!GC_all_interior_pointers){ GC_STATIC_ASSERT(VALID_OFFSET_SZ<=HBLKSIZE); GC_ASSERT(offset < VALID_OFFSET_SZ); GC_register_displacement(offset); } } result+=offset; GC_ASSERT((word)result % align==0); return result; } GC_API int GC_CALL GC_posix_memalign(void**memptr,size_t align,size_t lb) { size_t align_minus_one=align - 1; if (align < sizeof(void*)||(align_minus_one&align)!=0){ #ifdef MSWINCE return ERROR_INVALID_PARAMETER; #else return EINVAL; #endif } if ((*memptr=GC_memalign(align,lb))==NULL){ #ifdef MSWINCE return ERROR_NOT_ENOUGH_MEMORY; #else return ENOMEM; #endif } return 0; } GC_API GC_ATTR_MALLOC char*GC_CALL GC_strdup(const char*s) { char*copy; size_t lb; if (s==NULL)return NULL; lb=strlen(s)+1; copy=(char*)GC_malloc_atomic(lb); if (NULL==copy){ #ifndef MSWINCE errno=ENOMEM; #endif return NULL; } BCOPY(s,copy,lb); return copy; } GC_API GC_ATTR_MALLOC char*GC_CALL GC_strndup(const char*str,size_t size) { char*copy; size_t len=strlen(str); if (len > size) len=size; copy=(char*)GC_malloc_atomic(len+1); if (copy==NULL){ #ifndef MSWINCE errno=ENOMEM; #endif return NULL; } if (EXPECT(len > 0,TRUE)) BCOPY(str,copy,len); copy[len]='\0'; return copy; } #ifdef GC_REQUIRE_WCSDUP #include GC_API GC_ATTR_MALLOC wchar_t*GC_CALL GC_wcsdup(const wchar_t*str) { size_t lb=(wcslen(str)+1)*sizeof(wchar_t); wchar_t*copy=(wchar_t*)GC_malloc_atomic(lb); if (copy==NULL){ #ifndef MSWINCE errno=ENOMEM; #endif return NULL; } BCOPY(str,copy,lb); return copy; } #endif #ifndef CPPCHECK GC_API void*GC_CALL GC_malloc_stubborn(size_t lb) { return GC_malloc(lb); } GC_API void GC_CALL GC_change_stubborn(const void*p GC_ATTR_UNUSED) { } #endif GC_API void GC_CALL GC_end_stubborn_change(const void*p) { GC_dirty(p); } GC_API void GC_CALL GC_ptr_store_and_dirty(void*p,const void*q) { *(const void**)p=q; GC_dirty(p); REACHABLE_AFTER_DIRTY(q); } #if defined(__MINGW32__)&&!defined(__MINGW_EXCPT_DEFINE_PSDK)&&defined(__i386__) #define __MINGW_EXCPT_DEFINE_PSDK 1 #endif #include #if defined(MSWIN32)&&defined(__GNUC__) #include #endif GC_ATTR_NOINLINE void GC_noop6(word arg1 GC_ATTR_UNUSED,word arg2 GC_ATTR_UNUSED, word arg3 GC_ATTR_UNUSED,word arg4 GC_ATTR_UNUSED, word arg5 GC_ATTR_UNUSED,word arg6 GC_ATTR_UNUSED) { #if defined(AO_HAVE_compiler_barrier)&&!defined(BASE_ATOMIC_OPS_EMULATED) AO_compiler_barrier(); #else GC_noop1(0); #endif } volatile word GC_noop_sink; GC_ATTR_NO_SANITIZE_THREAD GC_API void GC_CALL GC_noop1(word x) { GC_noop_sink=x; } GC_INNER struct obj_kind GC_obj_kinds[MAXOBJKINDS]={ {&GC_aobjfreelist[0],0, GC_DS_LENGTH,FALSE,FALSE OK_DISCLAIM_INITZ }, {&GC_objfreelist[0],0, GC_DS_LENGTH, TRUE,TRUE OK_DISCLAIM_INITZ }, {&GC_uobjfreelist[0],0, GC_DS_LENGTH,TRUE,TRUE OK_DISCLAIM_INITZ }, #ifdef GC_ATOMIC_UNCOLLECTABLE {&GC_auobjfreelist[0],0, GC_DS_LENGTH,FALSE,FALSE OK_DISCLAIM_INITZ }, #endif }; #ifndef INITIAL_MARK_STACK_SIZE #define INITIAL_MARK_STACK_SIZE (1*HBLKSIZE) #endif #if!defined(GC_DISABLE_INCREMENTAL) STATIC word GC_n_rescuing_pages=0; #endif #ifdef PARALLEL_MARK GC_INNER GC_bool GC_parallel_mark_disabled=FALSE; #endif GC_INNER GC_bool GC_collection_in_progress(void) { return(GC_mark_state!=MS_NONE); } GC_INNER void GC_clear_hdr_marks(hdr*hhdr) { size_t last_bit; #ifdef AO_HAVE_load last_bit=FINAL_MARK_BIT((size_t)AO_load((volatile AO_t*)&hhdr->hb_sz)); #else last_bit=FINAL_MARK_BIT((size_t)hhdr->hb_sz); #endif BZERO(hhdr->hb_marks,sizeof(hhdr->hb_marks)); set_mark_bit_from_hdr(hhdr,last_bit); hhdr->hb_n_marks=0; } GC_INNER void GC_set_hdr_marks(hdr*hhdr) { unsigned i; size_t sz=(size_t)hhdr->hb_sz; unsigned n_marks=(unsigned)FINAL_MARK_BIT(sz); #ifdef USE_MARK_BYTES for (i=0;i<=n_marks;i+=(unsigned)MARK_BIT_OFFSET(sz)){ hhdr->hb_marks[i]=1; } #else for (i=0;i < divWORDSZ(n_marks+WORDSZ);++i){ hhdr->hb_marks[i]=GC_WORD_MAX; } #endif #ifdef MARK_BIT_PER_OBJ hhdr->hb_n_marks=n_marks; #else hhdr->hb_n_marks=HBLK_OBJS(sz); #endif } static void clear_marks_for_block(struct hblk*h,word dummy GC_ATTR_UNUSED) { hdr*hhdr=HDR(h); if (IS_UNCOLLECTABLE(hhdr->hb_obj_kind))return; GC_clear_hdr_marks(hhdr); } GC_API void GC_CALL GC_set_mark_bit(const void*p) { struct hblk*h=HBLKPTR(p); hdr*hhdr=HDR(h); word bit_no=MARK_BIT_NO((ptr_t)p - (ptr_t)h,hhdr->hb_sz); if (!mark_bit_from_hdr(hhdr,bit_no)){ set_mark_bit_from_hdr(hhdr,bit_no); ++hhdr->hb_n_marks; } } GC_API void GC_CALL GC_clear_mark_bit(const void*p) { struct hblk*h=HBLKPTR(p); hdr*hhdr=HDR(h); word bit_no=MARK_BIT_NO((ptr_t)p - (ptr_t)h,hhdr->hb_sz); if (mark_bit_from_hdr(hhdr,bit_no)){ size_t n_marks=hhdr->hb_n_marks; GC_ASSERT(n_marks!=0); clear_mark_bit_from_hdr(hhdr,bit_no); n_marks--; #ifdef PARALLEL_MARK if (n_marks!=0||!GC_parallel) hhdr->hb_n_marks=n_marks; #else hhdr->hb_n_marks=n_marks; #endif } } GC_API int GC_CALL GC_is_marked(const void*p) { struct hblk*h=HBLKPTR(p); hdr*hhdr=HDR(h); word bit_no=MARK_BIT_NO((ptr_t)p - (ptr_t)h,hhdr->hb_sz); return (int)mark_bit_from_hdr(hhdr,bit_no); } GC_INNER void GC_clear_marks(void) { GC_apply_to_all_blocks(clear_marks_for_block,(word)0); GC_objects_are_marked=FALSE; GC_mark_state=MS_INVALID; GC_scan_ptr=NULL; } GC_INNER void GC_initiate_gc(void) { GC_ASSERT(I_HOLD_LOCK()); #ifndef GC_DISABLE_INCREMENTAL if (GC_incremental){ #ifdef CHECKSUMS GC_read_dirty(FALSE); GC_check_dirty(); #else GC_read_dirty(GC_mark_state==MS_INVALID); #endif } GC_n_rescuing_pages=0; #endif if (GC_mark_state==MS_NONE){ GC_mark_state=MS_PUSH_RESCUERS; } else if (GC_mark_state!=MS_INVALID){ ABORT("Unexpected state"); } GC_scan_ptr=NULL; } #ifdef PARALLEL_MARK STATIC void GC_do_parallel_mark(void); #endif #ifdef GC_DISABLE_INCREMENTAL #define GC_push_next_marked_dirty(h)GC_push_next_marked(h) #else STATIC struct hblk*GC_push_next_marked_dirty(struct hblk*h); #endif STATIC struct hblk*GC_push_next_marked(struct hblk*h); STATIC struct hblk*GC_push_next_marked_uncollectable(struct hblk*h); static void alloc_mark_stack(size_t); #ifdef WRAP_MARK_SOME STATIC GC_bool GC_mark_some_inner(ptr_t cold_gc_frame) #else GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame) #endif { switch(GC_mark_state){ case MS_NONE: break; case MS_PUSH_RESCUERS: if ((word)GC_mark_stack_top >=(word)(GC_mark_stack_limit - INITIAL_MARK_STACK_SIZE/2)){ GC_mark_stack_too_small=TRUE; MARK_FROM_MARK_STACK(); break; } else { GC_scan_ptr=GC_push_next_marked_dirty(GC_scan_ptr); if (NULL==GC_scan_ptr){ #if!defined(GC_DISABLE_INCREMENTAL) GC_COND_LOG_PRINTF("Marked from %lu dirty pages\n", (unsigned long)GC_n_rescuing_pages); #endif GC_push_roots(FALSE,cold_gc_frame); GC_objects_are_marked=TRUE; if (GC_mark_state!=MS_INVALID){ GC_mark_state=MS_ROOTS_PUSHED; } } } break; case MS_PUSH_UNCOLLECTABLE: if ((word)GC_mark_stack_top >=(word)(GC_mark_stack+GC_mark_stack_size/4)){ #ifdef PARALLEL_MARK if (GC_parallel)GC_mark_stack_too_small=TRUE; #endif MARK_FROM_MARK_STACK(); break; } else { GC_scan_ptr=GC_push_next_marked_uncollectable(GC_scan_ptr); if (NULL==GC_scan_ptr){ GC_push_roots(TRUE,cold_gc_frame); GC_objects_are_marked=TRUE; if (GC_mark_state!=MS_INVALID){ GC_mark_state=MS_ROOTS_PUSHED; } } } break; case MS_ROOTS_PUSHED: #ifdef PARALLEL_MARK if (GC_parallel&&!GC_parallel_mark_disabled){ GC_do_parallel_mark(); GC_ASSERT((word)GC_mark_stack_top < (word)GC_first_nonempty); GC_mark_stack_top=GC_mark_stack - 1; if (GC_mark_stack_too_small){ alloc_mark_stack(2*GC_mark_stack_size); } if (GC_mark_state==MS_ROOTS_PUSHED){ GC_mark_state=MS_NONE; return(TRUE); } break; } #endif if ((word)GC_mark_stack_top>=(word)GC_mark_stack){ MARK_FROM_MARK_STACK(); break; } else { GC_mark_state=MS_NONE; if (GC_mark_stack_too_small){ alloc_mark_stack(2*GC_mark_stack_size); } return(TRUE); } case MS_INVALID: case MS_PARTIALLY_INVALID: if (!GC_objects_are_marked){ GC_mark_state=MS_PUSH_UNCOLLECTABLE; break; } if ((word)GC_mark_stack_top>=(word)GC_mark_stack){ MARK_FROM_MARK_STACK(); break; } if (NULL==GC_scan_ptr&&GC_mark_state==MS_INVALID){ if (GC_mark_stack_too_small){ alloc_mark_stack(2*GC_mark_stack_size); } GC_mark_state=MS_PARTIALLY_INVALID; } GC_scan_ptr=GC_push_next_marked(GC_scan_ptr); if (NULL==GC_scan_ptr&&GC_mark_state==MS_PARTIALLY_INVALID){ GC_push_roots(TRUE,cold_gc_frame); GC_objects_are_marked=TRUE; if (GC_mark_state!=MS_INVALID){ GC_mark_state=MS_ROOTS_PUSHED; } } break; default: ABORT("GC_mark_some:bad state"); } return(FALSE); } #ifdef WRAP_MARK_SOME #if (defined(MSWIN32)||defined(MSWINCE))&&defined(__GNUC__) typedef struct { EXCEPTION_REGISTRATION ex_reg; void*alt_path; } ext_ex_regn; static EXCEPTION_DISPOSITION mark_ex_handler( struct _EXCEPTION_RECORD*ex_rec, void*est_frame, struct _CONTEXT*context, void*disp_ctxt GC_ATTR_UNUSED) { if (ex_rec->ExceptionCode==STATUS_ACCESS_VIOLATION){ ext_ex_regn*xer=(ext_ex_regn*)est_frame; context->Esp=context->Ebp; context->Ebp=*((DWORD*)context->Esp); context->Esp=context->Esp - 8; context->Eip=(DWORD)(xer->alt_path); return ExceptionContinueExecution; } else { return ExceptionContinueSearch; } } #endif GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame) { GC_bool ret_val; #if defined(MSWIN32)||defined(MSWINCE) #ifndef __GNUC__ __try { ret_val=GC_mark_some_inner(cold_gc_frame); } __except (GetExceptionCode()==EXCEPTION_ACCESS_VIOLATION? EXCEPTION_EXECUTE_HANDLER:EXCEPTION_CONTINUE_SEARCH){ goto handle_ex; } #if defined(GC_WIN32_THREADS)&&!defined(GC_PTHREADS) if (GC_started_thread_while_stopped()) goto handle_thr_start; #endif rm_handler: return ret_val; #else ext_ex_regn er; #if GC_GNUC_PREREQ(4,7)||GC_CLANG_PREREQ(3,3) #pragma GCC diagnostic push #if defined(__clang__)||GC_GNUC_PREREQ(6,4) #pragma GCC diagnostic ignored "-Wpedantic" #else #pragma GCC diagnostic ignored "-pedantic" #endif er.alt_path=&&handle_ex; #pragma GCC diagnostic pop #elif!defined(CPPCHECK) er.alt_path=&&handle_ex; #endif er.ex_reg.handler=mark_ex_handler; __asm__ __volatile__ ("movl %%fs:0,%0":"=r" (er.ex_reg.prev)); __asm__ __volatile__ ("movl %0,%%fs:0"::"r" (&er)); ret_val=GC_mark_some_inner(cold_gc_frame); if (er.alt_path==0) goto handle_ex; #if defined(GC_WIN32_THREADS)&&!defined(GC_PTHREADS) if (GC_started_thread_while_stopped()) goto handle_thr_start; #endif rm_handler: __asm__ __volatile__ ("mov %0,%%fs:0"::"r" (er.ex_reg.prev)); return ret_val; #endif #else #ifndef DEFAULT_VDB if (GC_auto_incremental){ WARN("Incremental GC incompatible with/proc roots\n",0); } #endif GC_setup_temporary_fault_handler(); if(SETJMP(GC_jmp_buf)!=0)goto handle_ex; ret_val=GC_mark_some_inner(cold_gc_frame); rm_handler: GC_reset_fault_handler(); return ret_val; #endif handle_ex: { static word warned_gc_no; if (warned_gc_no!=GC_gc_no){ WARN("Caught ACCESS_VIOLATION in marker;" " memory mapping disappeared\n",0); warned_gc_no=GC_gc_no; } } #if (defined(MSWIN32)||defined(MSWINCE))&&defined(GC_WIN32_THREADS)&&!defined(GC_PTHREADS) handle_thr_start: #endif #ifdef REGISTER_LIBRARIES_EARLY START_WORLD(); GC_cond_register_dynamic_libraries(); STOP_WORLD(); #endif GC_invalidate_mark_state(); GC_scan_ptr=NULL; ret_val=FALSE; goto rm_handler; } #endif GC_INNER void GC_invalidate_mark_state(void) { GC_mark_state=MS_INVALID; GC_mark_stack_top=GC_mark_stack-1; } GC_INNER mse*GC_signal_mark_stack_overflow(mse*msp) { GC_mark_state=MS_INVALID; #ifdef PARALLEL_MARK if (!GC_parallel) GC_mark_stack_too_small=TRUE; #else GC_mark_stack_too_small=TRUE; #endif GC_COND_LOG_PRINTF("Mark stack overflow;current size=%lu entries\n", (unsigned long)GC_mark_stack_size); return(msp - GC_MARK_STACK_DISCARDS); } GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY GC_ATTR_NO_SANITIZE_THREAD GC_INNER mse*GC_mark_from(mse*mark_stack_top,mse*mark_stack, mse*mark_stack_limit) { signed_word credit=HBLKSIZE; ptr_t current_p; word current; ptr_t limit=0; word descr; ptr_t greatest_ha=(ptr_t)GC_greatest_plausible_heap_addr; ptr_t least_ha=(ptr_t)GC_least_plausible_heap_addr; DECLARE_HDR_CACHE; #define SPLIT_RANGE_WORDS 128 GC_objects_are_marked=TRUE; INIT_HDR_CACHE; #ifdef OS2 while ((word)mark_stack_top>=(word)mark_stack&&credit>=0) #else while (((((word)mark_stack_top - (word)mark_stack)|(word)credit) &SIGNB)==0) #endif { current_p=mark_stack_top->mse_start; descr=mark_stack_top->mse_descr.w; retry: if (descr&((~(WORDS_TO_BYTES(SPLIT_RANGE_WORDS)- 1))|GC_DS_TAGS)){ word tag=descr&GC_DS_TAGS; GC_STATIC_ASSERT(GC_DS_TAGS==0x3); switch(tag){ case GC_DS_LENGTH: GC_ASSERT(descr < (word)GC_greatest_plausible_heap_addr - (word)GC_least_plausible_heap_addr ||(word)(current_p+descr) <=(word)GC_least_plausible_heap_addr ||(word)current_p>=(word)GC_greatest_plausible_heap_addr); #ifdef PARALLEL_MARK #define SHARE_BYTES 2048 if (descr > SHARE_BYTES&&GC_parallel &&(word)mark_stack_top < (word)(mark_stack_limit - 1)){ word new_size=(descr/2)&~(word)(sizeof(word)-1); mark_stack_top->mse_start=current_p; mark_stack_top->mse_descr.w=new_size+sizeof(word); mark_stack_top++; #ifdef ENABLE_TRACE if ((word)GC_trace_addr>=(word)current_p &&(word)GC_trace_addr < (word)(current_p+descr)){ GC_log_printf("GC #%u:large section;start %p,len %lu," " splitting (parallel)at %p\n", (unsigned)GC_gc_no,(void*)current_p, (unsigned long)descr, (void*)(current_p+new_size)); } #endif current_p+=new_size; descr-=new_size; goto retry; } #endif mark_stack_top->mse_start= limit=current_p+WORDS_TO_BYTES(SPLIT_RANGE_WORDS-1); mark_stack_top->mse_descr.w= descr - WORDS_TO_BYTES(SPLIT_RANGE_WORDS-1); #ifdef ENABLE_TRACE if ((word)GC_trace_addr>=(word)current_p &&(word)GC_trace_addr < (word)(current_p+descr)){ GC_log_printf("GC #%u:large section;start %p,len %lu," " splitting at %p\n", (unsigned)GC_gc_no,(void*)current_p, (unsigned long)descr,(void*)limit); } #endif limit+=sizeof(word)- ALIGNMENT; break; case GC_DS_BITMAP: mark_stack_top--; #ifdef ENABLE_TRACE if ((word)GC_trace_addr>=(word)current_p &&(word)GC_trace_addr < (word)(current_p +WORDS_TO_BYTES(WORDSZ-2))){ GC_log_printf("GC #%u:tracing from %p bitmap descr %lu\n", (unsigned)GC_gc_no,(void*)current_p, (unsigned long)descr); } #endif descr&=~GC_DS_TAGS; credit-=WORDS_TO_BYTES(WORDSZ/2); while (descr!=0){ if ((descr&SIGNB)!=0){ current=*(word*)current_p; FIXUP_POINTER(current); if (current>=(word)least_ha&¤t < (word)greatest_ha){ PREFETCH((ptr_t)current); #ifdef ENABLE_TRACE if (GC_trace_addr==current_p){ GC_log_printf("GC #%u:considering(3)%p->%p\n", (unsigned)GC_gc_no,(void*)current_p, (void*)current); } #endif PUSH_CONTENTS((ptr_t)current,mark_stack_top, mark_stack_limit,current_p); } } descr<<=1; current_p+=sizeof(word); } continue; case GC_DS_PROC: mark_stack_top--; #ifdef ENABLE_TRACE if ((word)GC_trace_addr>=(word)current_p &&GC_base(current_p)!=0 &&GC_base(current_p)==GC_base(GC_trace_addr)){ GC_log_printf("GC #%u:tracing from %p,proc descr %lu\n", (unsigned)GC_gc_no,(void*)current_p, (unsigned long)descr); } #endif credit-=GC_PROC_BYTES; mark_stack_top=(*PROC(descr))((word*)current_p,mark_stack_top, mark_stack_limit,ENV(descr)); continue; case GC_DS_PER_OBJECT: if ((signed_word)descr>=0){ descr=*(word*)(current_p+descr - GC_DS_PER_OBJECT); } else { ptr_t type_descr=*(ptr_t*)current_p; if (EXPECT(0==type_descr,FALSE)){ mark_stack_top--; continue; } descr=*(word*)(type_descr - ((signed_word)descr+(GC_INDIR_PER_OBJ_BIAS - GC_DS_PER_OBJECT))); } if (0==descr){ mark_stack_top--; continue; } goto retry; } } else { mark_stack_top--; #ifndef SMALL_CONFIG if (descr < sizeof(word)) continue; #endif #ifdef ENABLE_TRACE if ((word)GC_trace_addr>=(word)current_p &&(word)GC_trace_addr < (word)(current_p+descr)){ GC_log_printf("GC #%u:small object;start %p,len %lu\n", (unsigned)GC_gc_no,(void*)current_p, (unsigned long)descr); } #endif limit=current_p+(word)descr; } GC_ASSERT(!((word)current_p&(ALIGNMENT-1))); credit-=limit - current_p; limit-=sizeof(word); { #define PREF_DIST 4 #ifndef SMALL_CONFIG word deferred; for(;;){ PREFETCH(limit - PREF_DIST*CACHE_LINE_SIZE); GC_ASSERT((word)limit>=(word)current_p); deferred=*(word*)limit; FIXUP_POINTER(deferred); limit-=ALIGNMENT; if (deferred>=(word)least_ha&&deferred < (word)greatest_ha){ PREFETCH((ptr_t)deferred); break; } if ((word)current_p > (word)limit)goto next_object; deferred=*(word*)limit; FIXUP_POINTER(deferred); limit-=ALIGNMENT; if (deferred>=(word)least_ha&&deferred < (word)greatest_ha){ PREFETCH((ptr_t)deferred); break; } if ((word)current_p > (word)limit)goto next_object; } #endif while ((word)current_p<=(word)limit){ current=*(word*)current_p; FIXUP_POINTER(current); PREFETCH(current_p+PREF_DIST*CACHE_LINE_SIZE); if (current>=(word)least_ha&¤t < (word)greatest_ha){ PREFETCH((ptr_t)current); #ifdef ENABLE_TRACE if (GC_trace_addr==current_p){ GC_log_printf("GC #%u:considering(1)%p->%p\n", (unsigned)GC_gc_no,(void*)current_p, (void*)current); } #endif PUSH_CONTENTS((ptr_t)current,mark_stack_top, mark_stack_limit,current_p); } current_p+=ALIGNMENT; } #ifndef SMALL_CONFIG #ifdef ENABLE_TRACE if (GC_trace_addr==current_p){ GC_log_printf("GC #%u:considering(2)%p->%p\n", (unsigned)GC_gc_no,(void*)current_p, (void*)deferred); } #endif PUSH_CONTENTS((ptr_t)deferred,mark_stack_top, mark_stack_limit,current_p); next_object:; #endif } } return mark_stack_top; } #ifdef PARALLEL_MARK STATIC GC_bool GC_help_wanted=FALSE; STATIC unsigned GC_helper_count=0; STATIC unsigned GC_active_count=0; GC_INNER word GC_mark_no=0; #ifdef LINT2 #define LOCAL_MARK_STACK_SIZE (HBLKSIZE/8) #else #define LOCAL_MARK_STACK_SIZE HBLKSIZE #endif GC_INNER void GC_wait_for_markers_init(void) { signed_word count; if (GC_markers_m1==0) return; #ifndef CAN_HANDLE_FORK GC_ASSERT(NULL==GC_main_local_mark_stack); #else if (NULL==GC_main_local_mark_stack) #endif { size_t bytes_to_get= ROUNDUP_PAGESIZE_IF_MMAP(LOCAL_MARK_STACK_SIZE*sizeof(mse)); GC_ASSERT(GC_page_size!=0); GC_main_local_mark_stack=(mse*)GET_MEM(bytes_to_get); if (NULL==GC_main_local_mark_stack) ABORT("Insufficient memory for main local_mark_stack"); GC_add_to_our_memory((ptr_t)GC_main_local_mark_stack,bytes_to_get); } GC_acquire_mark_lock(); GC_fl_builder_count+=GC_markers_m1; count=GC_fl_builder_count; GC_release_mark_lock(); if (count!=0){ GC_ASSERT(count > 0); GC_wait_for_reclaim(); } } STATIC mse*GC_steal_mark_stack(mse*low,mse*high,mse*local, unsigned max,mse**next) { mse*p; mse*top=local - 1; unsigned i=0; GC_ASSERT((word)high>=(word)(low - 1) &&(word)(high - low+1)<=GC_mark_stack_size); for (p=low;(word)p<=(word)high&&i<=max;++p){ word descr=(word)AO_load(&p->mse_descr.ao); if (descr!=0){ AO_store_release_write(&p->mse_descr.ao,0); ++top; top->mse_descr.w=descr; top->mse_start=p->mse_start; GC_ASSERT((descr&GC_DS_TAGS)!=GC_DS_LENGTH ||descr < (word)GC_greatest_plausible_heap_addr - (word)GC_least_plausible_heap_addr ||(word)(p->mse_start+descr) <=(word)GC_least_plausible_heap_addr ||(word)p->mse_start >=(word)GC_greatest_plausible_heap_addr); ++i; if ((descr&GC_DS_TAGS)==GC_DS_LENGTH)i+=(int)(descr>>8); } } *next=p; return top; } STATIC void GC_return_mark_stack(mse*low,mse*high) { mse*my_top; mse*my_start; size_t stack_size; if ((word)high < (word)low)return; stack_size=high - low+1; GC_acquire_mark_lock(); my_top=GC_mark_stack_top; my_start=my_top+1; if ((word)(my_start - GC_mark_stack+stack_size) > (word)GC_mark_stack_size){ GC_COND_LOG_PRINTF("No room to copy back mark stack\n"); GC_mark_state=MS_INVALID; GC_mark_stack_too_small=TRUE; } else { BCOPY(low,my_start,stack_size*sizeof(mse)); GC_ASSERT((mse*)AO_load((volatile AO_t*)(&GC_mark_stack_top)) ==my_top); AO_store_release_write((volatile AO_t*)(&GC_mark_stack_top), (AO_t)(my_top+stack_size)); } GC_release_mark_lock(); GC_notify_all_marker(); } #ifndef N_LOCAL_ITERS #define N_LOCAL_ITERS 1 #endif static GC_bool has_inactive_helpers(void) { GC_bool res; GC_acquire_mark_lock(); res=GC_active_count < GC_helper_count; GC_release_mark_lock(); return res; } STATIC void GC_do_local_mark(mse*local_mark_stack,mse*local_top) { unsigned n; for (;;){ for (n=0;n < N_LOCAL_ITERS;++n){ local_top=GC_mark_from(local_top,local_mark_stack, local_mark_stack+LOCAL_MARK_STACK_SIZE); if ((word)local_top < (word)local_mark_stack)return; if ((word)(local_top - local_mark_stack) >=LOCAL_MARK_STACK_SIZE/2){ GC_return_mark_stack(local_mark_stack,local_top); return; } } if ((word)AO_load((volatile AO_t*)&GC_mark_stack_top) < (word)AO_load(&GC_first_nonempty) &&(word)local_top > (word)(local_mark_stack+1) &&has_inactive_helpers()){ mse*new_bottom=local_mark_stack +(local_top - local_mark_stack)/2; GC_ASSERT((word)new_bottom > (word)local_mark_stack &&(word)new_bottom < (word)local_top); GC_return_mark_stack(local_mark_stack,new_bottom - 1); memmove(local_mark_stack,new_bottom, (local_top - new_bottom+1)*sizeof(mse)); local_top-=(new_bottom - local_mark_stack); } } } #ifndef ENTRIES_TO_GET #define ENTRIES_TO_GET 5 #endif STATIC void GC_mark_local(mse*local_mark_stack,int id) { mse*my_first_nonempty; GC_active_count++; my_first_nonempty=(mse*)AO_load(&GC_first_nonempty); GC_ASSERT((word)GC_mark_stack<=(word)my_first_nonempty); GC_ASSERT((word)my_first_nonempty <=(word)AO_load((volatile AO_t*)&GC_mark_stack_top)+sizeof(mse)); GC_VERBOSE_LOG_PRINTF("Starting mark helper %d\n",id); GC_release_mark_lock(); for (;;){ size_t n_on_stack; unsigned n_to_get; mse*my_top; mse*local_top; mse*global_first_nonempty=(mse*)AO_load(&GC_first_nonempty); GC_ASSERT((word)my_first_nonempty>=(word)GC_mark_stack&& (word)my_first_nonempty<= (word)AO_load((volatile AO_t*)&GC_mark_stack_top) +sizeof(mse)); GC_ASSERT((word)global_first_nonempty>=(word)GC_mark_stack); if ((word)my_first_nonempty < (word)global_first_nonempty){ my_first_nonempty=global_first_nonempty; } else if ((word)global_first_nonempty < (word)my_first_nonempty){ (void)AO_compare_and_swap(&GC_first_nonempty, (AO_t)global_first_nonempty, (AO_t)my_first_nonempty); } my_top=(mse*)AO_load_acquire((volatile AO_t*)(&GC_mark_stack_top)); if ((word)my_top < (word)my_first_nonempty){ GC_acquire_mark_lock(); my_top=GC_mark_stack_top; n_on_stack=my_top - my_first_nonempty+1; if (0==n_on_stack){ GC_active_count--; GC_ASSERT(GC_active_count<=GC_helper_count); if (0==GC_active_count)GC_notify_all_marker(); while (GC_active_count > 0 &&(word)AO_load(&GC_first_nonempty) > (word)GC_mark_stack_top){ GC_wait_marker(); } if (GC_active_count==0 &&(word)AO_load(&GC_first_nonempty) > (word)GC_mark_stack_top){ GC_bool need_to_notify=FALSE; GC_helper_count--; if (0==GC_helper_count)need_to_notify=TRUE; GC_VERBOSE_LOG_PRINTF("Finished mark helper %d\n",id); if (need_to_notify)GC_notify_all_marker(); return; } GC_active_count++; GC_ASSERT(GC_active_count > 0); GC_release_mark_lock(); continue; } else { GC_release_mark_lock(); } } else { n_on_stack=my_top - my_first_nonempty+1; } n_to_get=ENTRIES_TO_GET; if (n_on_stack < 2*ENTRIES_TO_GET)n_to_get=1; local_top=GC_steal_mark_stack(my_first_nonempty,my_top, local_mark_stack,n_to_get, &my_first_nonempty); GC_ASSERT((word)my_first_nonempty>=(word)GC_mark_stack&& (word)my_first_nonempty<= (word)AO_load((volatile AO_t*)&GC_mark_stack_top) +sizeof(mse)); GC_do_local_mark(local_mark_stack,local_top); } } STATIC void GC_do_parallel_mark(void) { GC_acquire_mark_lock(); GC_ASSERT(I_HOLD_LOCK()); if (GC_help_wanted||GC_active_count!=0||GC_helper_count!=0) ABORT("Tried to start parallel mark in bad state"); GC_VERBOSE_LOG_PRINTF("Starting marking for mark phase number %lu\n", (unsigned long)GC_mark_no); GC_first_nonempty=(AO_t)GC_mark_stack; GC_active_count=0; GC_helper_count=1; GC_help_wanted=TRUE; GC_notify_all_marker(); GC_mark_local(GC_main_local_mark_stack,0); GC_help_wanted=FALSE; while (GC_helper_count > 0){ GC_wait_marker(); } GC_VERBOSE_LOG_PRINTF("Finished marking for mark phase number %lu\n", (unsigned long)GC_mark_no); GC_mark_no++; GC_release_mark_lock(); GC_notify_all_marker(); } GC_INNER void GC_help_marker(word my_mark_no) { #define my_id my_id_mse.mse_descr.w mse my_id_mse; mse local_mark_stack[LOCAL_MARK_STACK_SIZE]; GC_ASSERT(GC_parallel); while (GC_mark_no < my_mark_no ||(!GC_help_wanted&&GC_mark_no==my_mark_no)){ GC_wait_marker(); } my_id=GC_helper_count; if (GC_mark_no!=my_mark_no||my_id > (unsigned)GC_markers_m1){ return; } GC_helper_count=(unsigned)my_id+1; GC_mark_local(local_mark_stack,(int)my_id); #undef my_id } #endif GC_INNER void GC_scratch_recycle_inner(void*ptr,size_t bytes) { if (ptr!=NULL){ size_t page_offset=(word)ptr&(GC_page_size - 1); size_t displ=0; size_t recycled_bytes; GC_ASSERT(bytes!=0); GC_ASSERT(GC_page_size!=0); if (page_offset!=0) displ=GC_page_size - page_offset; recycled_bytes=(bytes - displ)&~(GC_page_size - 1); GC_COND_LOG_PRINTF("Recycle %lu scratch-allocated bytes at %p\n", (unsigned long)recycled_bytes,ptr); if (recycled_bytes > 0) GC_add_to_heap((struct hblk*)((word)ptr+displ),recycled_bytes); } } static void alloc_mark_stack(size_t n) { mse*new_stack=(mse*)GC_scratch_alloc(n*sizeof(struct GC_ms_entry)); #ifdef GWW_VDB static GC_bool GC_incremental_at_stack_alloc=FALSE; GC_bool recycle_old=!GC_auto_incremental ||GC_incremental_at_stack_alloc; GC_incremental_at_stack_alloc=GC_auto_incremental; #else #define recycle_old TRUE #endif GC_mark_stack_too_small=FALSE; if (GC_mark_stack!=NULL){ if (new_stack!=0){ if (recycle_old){ GC_scratch_recycle_inner(GC_mark_stack, GC_mark_stack_size*sizeof(struct GC_ms_entry)); } GC_mark_stack=new_stack; GC_mark_stack_size=n; GC_mark_stack_limit=new_stack+n; GC_COND_LOG_PRINTF("Grew mark stack to %lu frames\n", (unsigned long)GC_mark_stack_size); } else { WARN("Failed to grow mark stack to %" WARN_PRIdPTR " frames\n",n); } } else if (NULL==new_stack){ GC_err_printf("No space for mark stack\n"); EXIT(); } else { GC_mark_stack=new_stack; GC_mark_stack_size=n; GC_mark_stack_limit=new_stack+n; } GC_mark_stack_top=GC_mark_stack-1; } GC_INNER void GC_mark_init(void) { alloc_mark_stack(INITIAL_MARK_STACK_SIZE); } GC_API void GC_CALL GC_push_all(void*bottom,void*top) { word length; bottom=(void*)(((word)bottom+ALIGNMENT-1)&~(ALIGNMENT-1)); top=(void*)((word)top&~(ALIGNMENT-1)); if ((word)bottom>=(word)top)return; GC_mark_stack_top++; if ((word)GC_mark_stack_top>=(word)GC_mark_stack_limit){ ABORT("Unexpected mark stack overflow"); } length=(word)top - (word)bottom; #if GC_DS_TAGS > ALIGNMENT - 1 length+=GC_DS_TAGS; length&=~GC_DS_TAGS; #endif GC_mark_stack_top->mse_start=(ptr_t)bottom; GC_mark_stack_top->mse_descr.w=length; } #ifndef GC_DISABLE_INCREMENTAL STATIC void GC_push_selected(ptr_t bottom,ptr_t top, GC_bool (*dirty_fn)(struct hblk*)) { struct hblk*h; bottom=(ptr_t)(((word)bottom+ALIGNMENT-1)&~(ALIGNMENT-1)); top=(ptr_t)(((word)top)&~(ALIGNMENT-1)); if ((word)bottom>=(word)top)return; h=HBLKPTR(bottom+HBLKSIZE); if ((word)top<=(word)h){ if ((*dirty_fn)(h-1)){ GC_push_all(bottom,top); } return; } if ((*dirty_fn)(h-1)){ if ((word)(GC_mark_stack_top - GC_mark_stack) > 3*GC_mark_stack_size/4){ GC_push_all(bottom,top); return; } GC_push_all(bottom,h); } while ((word)(h+1)<=(word)top){ if ((*dirty_fn)(h)){ if ((word)(GC_mark_stack_top - GC_mark_stack) > 3*GC_mark_stack_size/4){ GC_push_all(h,top); return; } else { GC_push_all(h,h+1); } } h++; } if ((ptr_t)h!=top&&(*dirty_fn)(h)){ GC_push_all(h,top); } } GC_API void GC_CALL GC_push_conditional(void*bottom,void*top,int all) { if (!all){ GC_push_selected((ptr_t)bottom,(ptr_t)top,GC_page_was_dirty); } else { #ifdef PROC_VDB if (GC_auto_incremental){ GC_push_selected((ptr_t)bottom,(ptr_t)top,GC_page_was_ever_dirty); } else #endif { GC_push_all(bottom,top); } } } #else GC_API void GC_CALL GC_push_conditional(void*bottom,void*top, int all GC_ATTR_UNUSED) { GC_push_all(bottom,top); } #endif #if defined(AMIGA)||defined(MACOS)||defined(GC_DARWIN_THREADS) void GC_push_one(word p) { GC_PUSH_ONE_STACK(p,MARKED_FROM_REGISTER); } #endif #ifdef GC_WIN32_THREADS GC_INNER void GC_push_many_regs(const word*regs,unsigned count) { unsigned i; for (i=0;i < count;i++) GC_PUSH_ONE_STACK(regs[i],MARKED_FROM_REGISTER); } #endif GC_API struct GC_ms_entry*GC_CALL GC_mark_and_push(void*obj, mse*mark_stack_ptr, mse*mark_stack_limit, void**src GC_ATTR_UNUSED) { hdr*hhdr; PREFETCH(obj); GET_HDR(obj,hhdr); if ((EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr),FALSE) &&(!GC_all_interior_pointers ||NULL==(hhdr=GC_find_header((ptr_t)GC_base(obj))))) ||EXPECT(HBLK_IS_FREE(hhdr),FALSE)){ GC_ADD_TO_BLACK_LIST_NORMAL(obj,(ptr_t)src); return mark_stack_ptr; } return GC_push_contents_hdr((ptr_t)obj,mark_stack_ptr,mark_stack_limit, (ptr_t)src,hhdr,TRUE); } #if defined(PRINT_BLACK_LIST)||defined(KEEP_BACK_PTRS) GC_INNER void GC_mark_and_push_stack(ptr_t p,ptr_t source) #else GC_INNER void GC_mark_and_push_stack(ptr_t p) #define source ((ptr_t)0) #endif { hdr*hhdr; ptr_t r=p; PREFETCH(p); GET_HDR(p,hhdr); if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr),FALSE)){ if (NULL==hhdr ||(r=(ptr_t)GC_base(p))==NULL ||(hhdr=HDR(r))==NULL){ GC_ADD_TO_BLACK_LIST_STACK(p,source); return; } } if (EXPECT(HBLK_IS_FREE(hhdr),FALSE)){ GC_ADD_TO_BLACK_LIST_NORMAL(p,source); return; } #ifdef THREADS GC_dirty(p); #endif GC_mark_stack_top=GC_push_contents_hdr(r,GC_mark_stack_top, GC_mark_stack_limit, source,hhdr,FALSE); } #undef source #ifdef TRACE_BUF #ifndef TRACE_ENTRIES #define TRACE_ENTRIES 1000 #endif struct trace_entry { char*kind; word gc_no; word bytes_allocd; word arg1; word arg2; } GC_trace_buf[TRACE_ENTRIES]={ { NULL,0,0,0,0 } }; void GC_add_trace_entry(char*kind,word arg1,word arg2) { GC_trace_buf[GC_trace_buf_ptr].kind=kind; GC_trace_buf[GC_trace_buf_ptr].gc_no=GC_gc_no; GC_trace_buf[GC_trace_buf_ptr].bytes_allocd=GC_bytes_allocd; GC_trace_buf[GC_trace_buf_ptr].arg1=arg1^0x80000000; GC_trace_buf[GC_trace_buf_ptr].arg2=arg2^0x80000000; GC_trace_buf_ptr++; if (GC_trace_buf_ptr>=TRACE_ENTRIES)GC_trace_buf_ptr=0; } GC_API void GC_CALL GC_print_trace_inner(word gc_no) { int i; for (i=GC_trace_buf_ptr-1;i!=GC_trace_buf_ptr;i--){ struct trace_entry*p; if (i < 0)i=TRACE_ENTRIES-1; p=GC_trace_buf+i; if (p->gc_no < gc_no||p->kind==0){ return; } GC_printf("Trace:%s (gc:%u,bytes:%lu)0x%lX,0x%lX\n", p->kind,(unsigned)p->gc_no, (unsigned long)p->bytes_allocd, (long)p->arg1^0x80000000L,(long)p->arg2^0x80000000L); } GC_printf("Trace incomplete\n"); } GC_API void GC_CALL GC_print_trace(word gc_no) { DCL_LOCK_STATE; LOCK(); GC_print_trace_inner(gc_no); UNLOCK(); } #endif GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY GC_ATTR_NO_SANITIZE_THREAD GC_API void GC_CALL GC_push_all_eager(void*bottom,void*top) { word*b=(word*)(((word)bottom+ALIGNMENT-1)&~(ALIGNMENT-1)); word*t=(word*)(((word)top)&~(ALIGNMENT-1)); REGISTER word*p; REGISTER word*lim; REGISTER ptr_t greatest_ha=(ptr_t)GC_greatest_plausible_heap_addr; REGISTER ptr_t least_ha=(ptr_t)GC_least_plausible_heap_addr; #define GC_greatest_plausible_heap_addr greatest_ha #define GC_least_plausible_heap_addr least_ha if (top==0)return; lim=t - 1; for (p=b;(word)p<=(word)lim; p=(word*)(((ptr_t)p)+ALIGNMENT)){ REGISTER word q=*p; GC_PUSH_ONE_STACK(q,p); } #undef GC_greatest_plausible_heap_addr #undef GC_least_plausible_heap_addr } GC_INNER void GC_push_all_stack(ptr_t bottom,ptr_t top) { #ifndef NEED_FIXUP_POINTER if (GC_all_interior_pointers #if defined(THREADS)&&defined(MPROTECT_VDB) &&!GC_auto_incremental #endif &&(word)GC_mark_stack_top < (word)(GC_mark_stack_limit - INITIAL_MARK_STACK_SIZE/8)){ GC_push_all(bottom,top); } else #endif { GC_push_all_eager(bottom,top); } } #if defined(WRAP_MARK_SOME)&&defined(PARALLEL_MARK) GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY GC_ATTR_NO_SANITIZE_THREAD GC_INNER void GC_push_conditional_eager(void*bottom,void*top, GC_bool all) { word*b=(word*)(((word)bottom+ALIGNMENT-1)&~(ALIGNMENT-1)); word*t=(word*)(((word)top)&~(ALIGNMENT-1)); REGISTER word*p; REGISTER word*lim; REGISTER ptr_t greatest_ha=(ptr_t)GC_greatest_plausible_heap_addr; REGISTER ptr_t least_ha=(ptr_t)GC_least_plausible_heap_addr; #define GC_greatest_plausible_heap_addr greatest_ha #define GC_least_plausible_heap_addr least_ha if (top==NULL) return; (void)all; lim=t - 1; for (p=b;(word)p<=(word)lim;p=(word*)((ptr_t)p+ALIGNMENT)){ REGISTER word q=*p; GC_PUSH_ONE_HEAP(q,p,GC_mark_stack_top); } #undef GC_greatest_plausible_heap_addr #undef GC_least_plausible_heap_addr } #endif #if!defined(SMALL_CONFIG)&&!defined(USE_MARK_BYTES)&&defined(MARK_BIT_PER_GRANULE) #if GC_GRANULE_WORDS==1 #define USE_PUSH_MARKED_ACCELERATORS #define PUSH_GRANULE(q)do { word qcontents=(q)[0];GC_PUSH_ONE_HEAP(qcontents,q,GC_mark_stack_top);} while (0) #elif GC_GRANULE_WORDS==2 #define USE_PUSH_MARKED_ACCELERATORS #define PUSH_GRANULE(q)do { word qcontents=(q)[0];GC_PUSH_ONE_HEAP(qcontents,q,GC_mark_stack_top);qcontents=(q)[1];GC_PUSH_ONE_HEAP(qcontents,(q)+1,GC_mark_stack_top);} while (0) #elif GC_GRANULE_WORDS==4 #define USE_PUSH_MARKED_ACCELERATORS #define PUSH_GRANULE(q)do { word qcontents=(q)[0];GC_PUSH_ONE_HEAP(qcontents,q,GC_mark_stack_top);qcontents=(q)[1];GC_PUSH_ONE_HEAP(qcontents,(q)+1,GC_mark_stack_top);qcontents=(q)[2];GC_PUSH_ONE_HEAP(qcontents,(q)+2,GC_mark_stack_top);qcontents=(q)[3];GC_PUSH_ONE_HEAP(qcontents,(q)+3,GC_mark_stack_top);} while (0) #endif #endif #ifdef USE_PUSH_MARKED_ACCELERATORS STATIC void GC_push_marked1(struct hblk*h,hdr*hhdr) { word*mark_word_addr=&(hhdr->hb_marks[0]); word*p; word*plim; ptr_t greatest_ha=(ptr_t)GC_greatest_plausible_heap_addr; ptr_t least_ha=(ptr_t)GC_least_plausible_heap_addr; mse*mark_stack_top=GC_mark_stack_top; mse*mark_stack_limit=GC_mark_stack_limit; #undef GC_mark_stack_top #undef GC_mark_stack_limit #define GC_mark_stack_top mark_stack_top #define GC_mark_stack_limit mark_stack_limit #define GC_greatest_plausible_heap_addr greatest_ha #define GC_least_plausible_heap_addr least_ha p=(word*)(h->hb_body); plim=(word*)(((word)h)+HBLKSIZE); while ((word)p < (word)plim){ word mark_word=*mark_word_addr++; word*q=p; while(mark_word!=0){ if (mark_word&1){ PUSH_GRANULE(q); } q+=GC_GRANULE_WORDS; mark_word>>=1; } p+=WORDSZ*GC_GRANULE_WORDS; } #undef GC_greatest_plausible_heap_addr #undef GC_least_plausible_heap_addr #undef GC_mark_stack_top #undef GC_mark_stack_limit #define GC_mark_stack_limit GC_arrays._mark_stack_limit #define GC_mark_stack_top GC_arrays._mark_stack_top GC_mark_stack_top=mark_stack_top; } #ifndef UNALIGNED_PTRS STATIC void GC_push_marked2(struct hblk*h,hdr*hhdr) { word*mark_word_addr=&(hhdr->hb_marks[0]); word*p; word*plim; ptr_t greatest_ha=(ptr_t)GC_greatest_plausible_heap_addr; ptr_t least_ha=(ptr_t)GC_least_plausible_heap_addr; mse*mark_stack_top=GC_mark_stack_top; mse*mark_stack_limit=GC_mark_stack_limit; #undef GC_mark_stack_top #undef GC_mark_stack_limit #define GC_mark_stack_top mark_stack_top #define GC_mark_stack_limit mark_stack_limit #define GC_greatest_plausible_heap_addr greatest_ha #define GC_least_plausible_heap_addr least_ha p=(word*)(h->hb_body); plim=(word*)(((word)h)+HBLKSIZE); while ((word)p < (word)plim){ word mark_word=*mark_word_addr++; word*q=p; while(mark_word!=0){ if (mark_word&1){ PUSH_GRANULE(q); PUSH_GRANULE(q+GC_GRANULE_WORDS); } q+=2*GC_GRANULE_WORDS; mark_word>>=2; } p+=WORDSZ*GC_GRANULE_WORDS; } #undef GC_greatest_plausible_heap_addr #undef GC_least_plausible_heap_addr #undef GC_mark_stack_top #undef GC_mark_stack_limit #define GC_mark_stack_limit GC_arrays._mark_stack_limit #define GC_mark_stack_top GC_arrays._mark_stack_top GC_mark_stack_top=mark_stack_top; } #if GC_GRANULE_WORDS < 4 STATIC void GC_push_marked4(struct hblk*h,hdr*hhdr) { word*mark_word_addr=&(hhdr->hb_marks[0]); word*p; word*plim; ptr_t greatest_ha=(ptr_t)GC_greatest_plausible_heap_addr; ptr_t least_ha=(ptr_t)GC_least_plausible_heap_addr; mse*mark_stack_top=GC_mark_stack_top; mse*mark_stack_limit=GC_mark_stack_limit; #undef GC_mark_stack_top #undef GC_mark_stack_limit #define GC_mark_stack_top mark_stack_top #define GC_mark_stack_limit mark_stack_limit #define GC_greatest_plausible_heap_addr greatest_ha #define GC_least_plausible_heap_addr least_ha p=(word*)(h->hb_body); plim=(word*)(((word)h)+HBLKSIZE); while ((word)p < (word)plim){ word mark_word=*mark_word_addr++; word*q=p; while(mark_word!=0){ if (mark_word&1){ PUSH_GRANULE(q); PUSH_GRANULE(q+GC_GRANULE_WORDS); PUSH_GRANULE(q+2*GC_GRANULE_WORDS); PUSH_GRANULE(q+3*GC_GRANULE_WORDS); } q+=4*GC_GRANULE_WORDS; mark_word>>=4; } p+=WORDSZ*GC_GRANULE_WORDS; } #undef GC_greatest_plausible_heap_addr #undef GC_least_plausible_heap_addr #undef GC_mark_stack_top #undef GC_mark_stack_limit #define GC_mark_stack_limit GC_arrays._mark_stack_limit #define GC_mark_stack_top GC_arrays._mark_stack_top GC_mark_stack_top=mark_stack_top; } #endif #endif #endif STATIC void GC_push_marked(struct hblk*h,hdr*hhdr) { word sz=hhdr->hb_sz; word descr=hhdr->hb_descr; ptr_t p; word bit_no; ptr_t lim; mse*GC_mark_stack_top_reg; mse*mark_stack_limit=GC_mark_stack_limit; if (( GC_DS_LENGTH)==descr)return; if (GC_block_empty(hhdr))return; #if!defined(GC_DISABLE_INCREMENTAL) GC_n_rescuing_pages++; #endif GC_objects_are_marked=TRUE; if (sz > MAXOBJBYTES){ lim=h->hb_body; } else { lim=(ptr_t)((word)(h+1)->hb_body - sz); } switch(BYTES_TO_GRANULES(sz)){ #if defined(USE_PUSH_MARKED_ACCELERATORS) case 1: GC_push_marked1(h,hhdr); break; #if!defined(UNALIGNED_PTRS) case 2: GC_push_marked2(h,hhdr); break; #if GC_GRANULE_WORDS < 4 case 4: GC_push_marked4(h,hhdr); break; #endif #endif #else case 1: #endif default: GC_mark_stack_top_reg=GC_mark_stack_top; for (p=h->hb_body,bit_no=0;(word)p<=(word)lim; p+=sz,bit_no+=MARK_BIT_OFFSET(sz)){ if (mark_bit_from_hdr(hhdr,bit_no)){ GC_mark_stack_top_reg=GC_push_obj(p,hhdr,GC_mark_stack_top_reg, mark_stack_limit); } } GC_mark_stack_top=GC_mark_stack_top_reg; } } #ifdef ENABLE_DISCLAIM STATIC void GC_push_unconditionally(struct hblk*h,hdr*hhdr) { word sz=hhdr->hb_sz; word descr=hhdr->hb_descr; ptr_t p; ptr_t lim; mse*GC_mark_stack_top_reg; mse*mark_stack_limit=GC_mark_stack_limit; if (( GC_DS_LENGTH)==descr) return; #if!defined(GC_DISABLE_INCREMENTAL) GC_n_rescuing_pages++; #endif GC_objects_are_marked=TRUE; if (sz > MAXOBJBYTES) lim=h->hb_body; else lim=(ptr_t)((word)(h+1)->hb_body - sz); GC_mark_stack_top_reg=GC_mark_stack_top; for (p=h->hb_body;(word)p<=(word)lim;p+=sz) if ((*(word*)p&0x3)!=0) GC_mark_stack_top_reg=GC_push_obj(p,hhdr,GC_mark_stack_top_reg, mark_stack_limit); GC_mark_stack_top=GC_mark_stack_top_reg; } #endif #ifndef GC_DISABLE_INCREMENTAL STATIC GC_bool GC_block_was_dirty(struct hblk*h,hdr*hhdr) { word sz=hhdr->hb_sz; if (sz<=MAXOBJBYTES){ return(GC_page_was_dirty(h)); } else { ptr_t p=(ptr_t)h; while ((word)p < (word)h+sz){ if (GC_page_was_dirty((struct hblk*)p))return(TRUE); p+=HBLKSIZE; } return(FALSE); } } #endif STATIC struct hblk*GC_push_next_marked(struct hblk*h) { hdr*hhdr=HDR(h); if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr)||HBLK_IS_FREE(hhdr),FALSE)){ h=GC_next_block(h,FALSE); if (NULL==h)return NULL; hhdr=GC_find_header((ptr_t)h); } else { #ifdef LINT2 if (NULL==h)ABORT("Bad HDR()definition"); #endif } GC_push_marked(h,hhdr); return(h+OBJ_SZ_TO_BLOCKS(hhdr->hb_sz)); } #ifndef GC_DISABLE_INCREMENTAL STATIC struct hblk*GC_push_next_marked_dirty(struct hblk*h) { hdr*hhdr=HDR(h); if (!GC_incremental)ABORT("Dirty bits not set up"); for (;;){ if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr) ||HBLK_IS_FREE(hhdr),FALSE)){ h=GC_next_block(h,FALSE); if (NULL==h)return NULL; hhdr=GC_find_header((ptr_t)h); } else { #ifdef LINT2 if (NULL==h)ABORT("Bad HDR()definition"); #endif } if (GC_block_was_dirty(h,hhdr)) break; h+=OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); hhdr=HDR(h); } #ifdef ENABLE_DISCLAIM if ((hhdr->hb_flags&MARK_UNCONDITIONALLY)!=0){ GC_push_unconditionally(h,hhdr); } else #endif { GC_push_marked(h,hhdr); } return(h+OBJ_SZ_TO_BLOCKS(hhdr->hb_sz)); } #endif STATIC struct hblk*GC_push_next_marked_uncollectable(struct hblk*h) { hdr*hhdr=HDR(h); for (;;){ if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr) ||HBLK_IS_FREE(hhdr),FALSE)){ h=GC_next_block(h,FALSE); if (NULL==h)return NULL; hhdr=GC_find_header((ptr_t)h); } else { #ifdef LINT2 if (NULL==h)ABORT("Bad HDR()definition"); #endif } if (hhdr->hb_obj_kind==UNCOLLECTABLE){ GC_push_marked(h,hhdr); break; } #ifdef ENABLE_DISCLAIM if ((hhdr->hb_flags&MARK_UNCONDITIONALLY)!=0){ GC_push_unconditionally(h,hhdr); break; } #endif h+=OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); hhdr=HDR(h); } return(h+OBJ_SZ_TO_BLOCKS(hhdr->hb_sz)); } #include int GC_no_dls=0; #if!defined(NO_DEBUGGING)||defined(GC_ASSERTIONS) GC_INNER word GC_compute_root_size(void) { int i; word size=0; for (i=0;i < n_root_sets;i++){ size+=GC_static_roots[i].r_end - GC_static_roots[i].r_start; } return size; } #endif #if!defined(NO_DEBUGGING) void GC_print_static_roots(void) { int i; word size; for (i=0;i < n_root_sets;i++){ GC_printf("From %p to %p%s\n", (void*)GC_static_roots[i].r_start, (void*)GC_static_roots[i].r_end, GC_static_roots[i].r_tmp?" (temporary)":""); } GC_printf("GC_root_size:%lu\n",(unsigned long)GC_root_size); if ((size=GC_compute_root_size())!=GC_root_size) GC_err_printf("GC_root_size incorrect!!Should be:%lu\n", (unsigned long)size); } #endif #ifndef THREADS GC_INNER GC_bool GC_is_static_root(void*p) { static int last_root_set=MAX_ROOT_SETS; int i; if (last_root_set < n_root_sets &&(word)p>=(word)GC_static_roots[last_root_set].r_start &&(word)p < (word)GC_static_roots[last_root_set].r_end) return(TRUE); for (i=0;i < n_root_sets;i++){ if ((word)p>=(word)GC_static_roots[i].r_start &&(word)p < (word)GC_static_roots[i].r_end){ last_root_set=i; return(TRUE); } } return(FALSE); } #endif #if!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(CYGWIN32) GC_INLINE int rt_hash(ptr_t addr) { word result=(word)addr; #if CPP_WORDSZ > 8*LOG_RT_SIZE result^=result>>8*LOG_RT_SIZE; #endif #if CPP_WORDSZ > 4*LOG_RT_SIZE result^=result>>4*LOG_RT_SIZE; #endif result^=result>>2*LOG_RT_SIZE; result^=result>>LOG_RT_SIZE; result&=(RT_SIZE-1); return(result); } GC_INNER void*GC_roots_present(ptr_t b) { int h=rt_hash(b); struct roots*p=GC_root_index[h]; while (p!=0){ if (p->r_start==(ptr_t)b)return(p); p=p->r_next; } return NULL; } GC_INLINE void add_roots_to_index(struct roots*p) { int h=rt_hash(p->r_start); p->r_next=GC_root_index[h]; GC_root_index[h]=p; } #endif GC_INNER word GC_root_size=0; GC_API void GC_CALL GC_add_roots(void*b,void*e) { DCL_LOCK_STATE; if (!EXPECT(GC_is_initialized,TRUE))GC_init(); LOCK(); GC_add_roots_inner((ptr_t)b,(ptr_t)e,FALSE); UNLOCK(); } void GC_add_roots_inner(ptr_t b,ptr_t e,GC_bool tmp) { GC_ASSERT((word)b<=(word)e); b=(ptr_t)(((word)b+(sizeof(word)- 1))&~(word)(sizeof(word)- 1)); e=(ptr_t)((word)e&~(word)(sizeof(word)- 1)); if ((word)b>=(word)e)return; #if defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32) { int i; struct roots*old=NULL; for (i=0;i < n_root_sets;i++){ old=GC_static_roots+i; if ((word)b<=(word)old->r_end &&(word)e>=(word)old->r_start){ if ((word)b < (word)old->r_start){ GC_root_size+=old->r_start - b; old->r_start=b; } if ((word)e > (word)old->r_end){ GC_root_size+=e - old->r_end; old->r_end=e; } old->r_tmp&=tmp; break; } } if (i < n_root_sets){ struct roots*other; for (i++;i < n_root_sets;i++){ other=GC_static_roots+i; b=other->r_start; e=other->r_end; if ((word)b<=(word)old->r_end &&(word)e>=(word)old->r_start){ if ((word)b < (word)old->r_start){ GC_root_size+=old->r_start - b; old->r_start=b; } if ((word)e > (word)old->r_end){ GC_root_size+=e - old->r_end; old->r_end=e; } old->r_tmp&=other->r_tmp; GC_root_size-=(other->r_end - other->r_start); other->r_start=GC_static_roots[n_root_sets-1].r_start; other->r_end=GC_static_roots[n_root_sets-1].r_end; n_root_sets--; } } return; } } #else { struct roots*old=(struct roots*)GC_roots_present(b); if (old!=0){ if ((word)e<=(word)old->r_end){ old->r_tmp&=tmp; return; } if (old->r_tmp==tmp||!tmp){ GC_root_size+=e - old->r_end; old->r_end=e; old->r_tmp=tmp; return; } b=old->r_end; } } #endif if (n_root_sets==MAX_ROOT_SETS){ ABORT("Too many root sets"); } #ifdef DEBUG_ADD_DEL_ROOTS GC_log_printf("Adding data root section %d:%p .. %p%s\n", n_root_sets,(void*)b,(void*)e, tmp?" (temporary)":""); #endif GC_static_roots[n_root_sets].r_start=(ptr_t)b; GC_static_roots[n_root_sets].r_end=(ptr_t)e; GC_static_roots[n_root_sets].r_tmp=tmp; #if!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(CYGWIN32) GC_static_roots[n_root_sets].r_next=0; add_roots_to_index(GC_static_roots+n_root_sets); #endif GC_root_size+=e - b; n_root_sets++; } GC_API void GC_CALL GC_clear_roots(void) { DCL_LOCK_STATE; if (!EXPECT(GC_is_initialized,TRUE))GC_init(); LOCK(); #ifdef THREADS GC_roots_were_cleared=TRUE; #endif n_root_sets=0; GC_root_size=0; #if!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(CYGWIN32) BZERO(GC_root_index,RT_SIZE*sizeof(void*)); #endif #ifdef DEBUG_ADD_DEL_ROOTS GC_log_printf("Clear all data root sections\n"); #endif UNLOCK(); } STATIC void GC_remove_root_at_pos(int i) { #ifdef DEBUG_ADD_DEL_ROOTS GC_log_printf("Remove data root section at %d:%p .. %p%s\n", i,(void*)GC_static_roots[i].r_start, (void*)GC_static_roots[i].r_end, GC_static_roots[i].r_tmp?" (temporary)":""); #endif GC_root_size-=(GC_static_roots[i].r_end - GC_static_roots[i].r_start); GC_static_roots[i].r_start=GC_static_roots[n_root_sets-1].r_start; GC_static_roots[i].r_end=GC_static_roots[n_root_sets-1].r_end; GC_static_roots[i].r_tmp=GC_static_roots[n_root_sets-1].r_tmp; n_root_sets--; } #if!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(CYGWIN32) STATIC void GC_rebuild_root_index(void) { int i; BZERO(GC_root_index,RT_SIZE*sizeof(void*)); for (i=0;i < n_root_sets;i++) add_roots_to_index(GC_static_roots+i); } #endif #if defined(DYNAMIC_LOADING)||defined(MSWIN32)||defined(MSWINCE)||defined(PCR)||defined(CYGWIN32) STATIC void GC_remove_tmp_roots(void) { int i; #if!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(CYGWIN32) int old_n_roots=n_root_sets; #endif for (i=0;i < n_root_sets;){ if (GC_static_roots[i].r_tmp){ GC_remove_root_at_pos(i); } else { i++; } } #if!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(CYGWIN32) if (n_root_sets < old_n_roots) GC_rebuild_root_index(); #endif } #endif #if!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(CYGWIN32) STATIC void GC_remove_roots_inner(ptr_t b,ptr_t e); GC_API void GC_CALL GC_remove_roots(void*b,void*e) { DCL_LOCK_STATE; if ((((word)b+(sizeof(word)- 1))&~(word)(sizeof(word)- 1))>= ((word)e&~(word)(sizeof(word)- 1))) return; LOCK(); GC_remove_roots_inner((ptr_t)b,(ptr_t)e); UNLOCK(); } STATIC void GC_remove_roots_inner(ptr_t b,ptr_t e) { int i; GC_bool rebuild=FALSE; for (i=0;i < n_root_sets;){ if ((word)GC_static_roots[i].r_start>=(word)b &&(word)GC_static_roots[i].r_end<=(word)e){ GC_remove_root_at_pos(i); rebuild=TRUE; } else { i++; } } if (rebuild) GC_rebuild_root_index(); } #endif #ifdef USE_PROC_FOR_LIBRARIES GC_INLINE void swap_static_roots(int i,int j) { ptr_t r_start=GC_static_roots[i].r_start; ptr_t r_end=GC_static_roots[i].r_end; GC_bool r_tmp=GC_static_roots[i].r_tmp; GC_static_roots[i].r_start=GC_static_roots[j].r_start; GC_static_roots[i].r_end=GC_static_roots[j].r_end; GC_static_roots[i].r_tmp=GC_static_roots[j].r_tmp; GC_static_roots[j].r_start=r_start; GC_static_roots[j].r_end=r_end; GC_static_roots[j].r_tmp=r_tmp; } GC_INNER void GC_remove_roots_subregion(ptr_t b,ptr_t e) { int i; GC_bool rebuild=FALSE; GC_ASSERT(I_HOLD_LOCK()); GC_ASSERT((word)b % sizeof(word)==0&&(word)e % sizeof(word)==0); for (i=0;i < n_root_sets;i++){ ptr_t r_start,r_end; if (GC_static_roots[i].r_tmp){ #ifdef GC_ASSERTIONS int j; for (j=i+1;j < n_root_sets;j++){ GC_ASSERT(GC_static_roots[j].r_tmp); } #endif break; } r_start=GC_static_roots[i].r_start; r_end=GC_static_roots[i].r_end; if (!EXPECT((word)e<=(word)r_start||(word)r_end<=(word)b,TRUE)){ #ifdef DEBUG_ADD_DEL_ROOTS GC_log_printf("Removing %p .. %p from root section %d (%p .. %p)\n", (void*)b,(void*)e, i,(void*)r_start,(void*)r_end); #endif if ((word)r_start < (word)b){ GC_root_size-=r_end - b; GC_static_roots[i].r_end=b; if ((word)e < (word)r_end){ int j; if (rebuild){ GC_rebuild_root_index(); rebuild=FALSE; } GC_add_roots_inner(e,r_end,FALSE); for (j=i+1;j < n_root_sets;j++) if (GC_static_roots[j].r_tmp) break; if (j < n_root_sets-1&&!GC_static_roots[n_root_sets-1].r_tmp){ swap_static_roots(j,n_root_sets - 1); rebuild=TRUE; } } } else { if ((word)e < (word)r_end){ GC_root_size-=e - r_start; GC_static_roots[i].r_start=e; } else { GC_remove_root_at_pos(i); if (i < n_root_sets - 1&&GC_static_roots[i].r_tmp &&!GC_static_roots[i+1].r_tmp){ int j; for (j=i+2;j < n_root_sets;j++) if (GC_static_roots[j].r_tmp) break; swap_static_roots(i,j - 1); } i--; } rebuild=TRUE; } } } if (rebuild) GC_rebuild_root_index(); } #endif #if!defined(NO_DEBUGGING) GC_API int GC_CALL GC_is_tmp_root(void*p) { static int last_root_set=MAX_ROOT_SETS; int i; if (last_root_set < n_root_sets &&(word)p>=(word)GC_static_roots[last_root_set].r_start &&(word)p < (word)GC_static_roots[last_root_set].r_end) return GC_static_roots[last_root_set].r_tmp; for (i=0;i < n_root_sets;i++){ if ((word)p>=(word)GC_static_roots[i].r_start &&(word)p < (word)GC_static_roots[i].r_end){ last_root_set=i; return GC_static_roots[i].r_tmp; } } return(FALSE); } #endif GC_INNER ptr_t GC_approx_sp(void) { volatile word sp; #if defined(S390)&&!defined(CPPCHECK)&&(__clang_major__ < 8) sp=(word)&sp; #elif defined(CPPCHECK)||(__GNUC__>=4&&!defined(STACK_NOT_SCANNED)) sp=(word)__builtin_frame_address(0); #else sp=(word)&sp; #endif return((ptr_t)sp); } GC_API void GC_CALL GC_clear_exclusion_table(void) { GC_excl_table_entries=0; } STATIC struct exclusion*GC_next_exclusion(ptr_t start_addr) { size_t low=0; size_t high; GC_ASSERT(GC_excl_table_entries > 0); high=GC_excl_table_entries - 1; while (high > low){ size_t mid=(low+high)>>1; if ((word)GC_excl_table[mid].e_end<=(word)start_addr){ low=mid+1; } else { high=mid; } } if ((word)GC_excl_table[low].e_end<=(word)start_addr)return 0; return GC_excl_table+low; } GC_INNER void GC_exclude_static_roots_inner(void*start,void*finish) { struct exclusion*next; size_t next_index; GC_ASSERT((word)start % sizeof(word)==0); GC_ASSERT((word)start < (word)finish); if (0==GC_excl_table_entries){ next=0; } else { next=GC_next_exclusion((ptr_t)start); } if (0!=next){ size_t i; if ((word)(next->e_start)< (word)finish){ ABORT("Exclusion ranges overlap"); } if ((word)(next->e_start)==(word)finish){ next->e_start=(ptr_t)start; return; } next_index=next - GC_excl_table; for (i=GC_excl_table_entries;i > next_index;--i){ GC_excl_table[i]=GC_excl_table[i-1]; } } else { next_index=GC_excl_table_entries; } if (GC_excl_table_entries==MAX_EXCLUSIONS)ABORT("Too many exclusions"); GC_excl_table[next_index].e_start=(ptr_t)start; GC_excl_table[next_index].e_end=(ptr_t)finish; ++GC_excl_table_entries; } GC_API void GC_CALL GC_exclude_static_roots(void*b,void*e) { DCL_LOCK_STATE; if (b==e)return; b=(void*)((word)b&~(word)(sizeof(word)- 1)); e=(void*)(((word)e+(sizeof(word)- 1))&~(word)(sizeof(word)- 1)); if (NULL==e) e=(void*)(~(word)(sizeof(word)- 1)); LOCK(); GC_exclude_static_roots_inner(b,e); UNLOCK(); } #if defined(WRAP_MARK_SOME)&&defined(PARALLEL_MARK) #define GC_PUSH_CONDITIONAL(b,t,all)(GC_parallel?GC_push_conditional_eager(b,t,all):GC_push_conditional(b,t,all)) #elif defined(GC_DISABLE_INCREMENTAL) #define GC_PUSH_CONDITIONAL(b,t,all)GC_push_all(b,t) #else #define GC_PUSH_CONDITIONAL(b,t,all)GC_push_conditional(b,t,all) #endif STATIC void GC_push_conditional_with_exclusions(ptr_t bottom,ptr_t top, GC_bool all GC_ATTR_UNUSED) { while ((word)bottom < (word)top){ struct exclusion*next=GC_next_exclusion(bottom); ptr_t excl_start; if (0==next ||(word)(excl_start=next->e_start)>=(word)top){ GC_PUSH_CONDITIONAL(bottom,top,all); break; } if ((word)excl_start > (word)bottom) GC_PUSH_CONDITIONAL(bottom,excl_start,all); bottom=next->e_end; } } #ifdef IA64 GC_INNER void GC_push_all_register_sections(ptr_t bs_lo,ptr_t bs_hi, int eager,struct GC_traced_stack_sect_s*traced_stack_sect) { while (traced_stack_sect!=NULL){ ptr_t frame_bs_lo=traced_stack_sect->backing_store_end; GC_ASSERT((word)frame_bs_lo<=(word)bs_hi); if (eager){ GC_push_all_eager(frame_bs_lo,bs_hi); } else { GC_push_all_stack(frame_bs_lo,bs_hi); } bs_hi=traced_stack_sect->saved_backing_store_ptr; traced_stack_sect=traced_stack_sect->prev; } GC_ASSERT((word)bs_lo<=(word)bs_hi); if (eager){ GC_push_all_eager(bs_lo,bs_hi); } else { GC_push_all_stack(bs_lo,bs_hi); } } #endif #ifdef THREADS GC_INNER void GC_push_all_stack_sections(ptr_t lo,ptr_t hi, struct GC_traced_stack_sect_s*traced_stack_sect) { while (traced_stack_sect!=NULL){ GC_ASSERT((word)lo HOTTER_THAN (word)traced_stack_sect); #ifdef STACK_GROWS_UP GC_push_all_stack((ptr_t)traced_stack_sect,lo); #else GC_push_all_stack(lo,(ptr_t)traced_stack_sect); #endif lo=traced_stack_sect->saved_stack_ptr; GC_ASSERT(lo!=NULL); traced_stack_sect=traced_stack_sect->prev; } GC_ASSERT(!((word)hi HOTTER_THAN (word)lo)); #ifdef STACK_GROWS_UP GC_push_all_stack(hi,lo); #else GC_push_all_stack(lo,hi); #endif } #else STATIC void GC_push_all_stack_partially_eager(ptr_t bottom,ptr_t top, ptr_t cold_gc_frame) { #ifndef NEED_FIXUP_POINTER if (GC_all_interior_pointers){ if (0==cold_gc_frame){ GC_push_all_stack(bottom,top); return; } GC_ASSERT((word)bottom<=(word)cold_gc_frame &&(word)cold_gc_frame<=(word)top); #ifdef STACK_GROWS_DOWN GC_push_all(cold_gc_frame - sizeof(ptr_t),top); GC_push_all_eager(bottom,cold_gc_frame); #else GC_push_all(bottom,cold_gc_frame+sizeof(ptr_t)); GC_push_all_eager(cold_gc_frame,top); #endif } else #endif { GC_push_all_eager(bottom,top); } #ifdef TRACE_BUF GC_add_trace_entry("GC_push_all_stack",(word)bottom,(word)top); #endif } STATIC void GC_push_all_stack_part_eager_sections(ptr_t lo,ptr_t hi, ptr_t cold_gc_frame,struct GC_traced_stack_sect_s*traced_stack_sect) { GC_ASSERT(traced_stack_sect==NULL||cold_gc_frame==NULL|| (word)cold_gc_frame HOTTER_THAN (word)traced_stack_sect); while (traced_stack_sect!=NULL){ GC_ASSERT((word)lo HOTTER_THAN (word)traced_stack_sect); #ifdef STACK_GROWS_UP GC_push_all_stack_partially_eager((ptr_t)traced_stack_sect,lo, cold_gc_frame); #else GC_push_all_stack_partially_eager(lo,(ptr_t)traced_stack_sect, cold_gc_frame); #endif lo=traced_stack_sect->saved_stack_ptr; GC_ASSERT(lo!=NULL); traced_stack_sect=traced_stack_sect->prev; cold_gc_frame=NULL; } GC_ASSERT(!((word)hi HOTTER_THAN (word)lo)); #ifdef STACK_GROWS_UP GC_push_all_stack_partially_eager(hi,lo,cold_gc_frame); #else GC_push_all_stack_partially_eager(lo,hi,cold_gc_frame); #endif } #endif STATIC void GC_push_current_stack(ptr_t cold_gc_frame, void*context GC_ATTR_UNUSED) { #if defined(THREADS) #ifdef STACK_GROWS_DOWN GC_push_all_eager(GC_approx_sp(),cold_gc_frame); #else GC_push_all_eager(cold_gc_frame,GC_approx_sp()); #endif #else GC_push_all_stack_part_eager_sections(GC_approx_sp(),GC_stackbottom, cold_gc_frame,GC_traced_stack_sect); #ifdef IA64 { ptr_t bsp=GC_save_regs_ret_val; ptr_t cold_gc_bs_pointer=bsp - 2048; if (GC_all_interior_pointers &&(word)cold_gc_bs_pointer > (word)BACKING_STORE_BASE){ if (GC_traced_stack_sect!=NULL &&(word)cold_gc_bs_pointer < (word)GC_traced_stack_sect->backing_store_end) cold_gc_bs_pointer= GC_traced_stack_sect->backing_store_end; GC_push_all_register_sections(BACKING_STORE_BASE, cold_gc_bs_pointer,FALSE,GC_traced_stack_sect); GC_push_all_eager(cold_gc_bs_pointer,bsp); } else { GC_push_all_register_sections(BACKING_STORE_BASE,bsp, TRUE,GC_traced_stack_sect); } } #endif #endif } GC_INNER void (*GC_push_typed_structures)(void)=0; GC_INNER void GC_cond_register_dynamic_libraries(void) { #if (defined(DYNAMIC_LOADING)&&!defined(MSWIN_XBOX1))||defined(CYGWIN32)||defined(MSWIN32)||defined(MSWINCE)||defined(PCR) GC_remove_tmp_roots(); if (!GC_no_dls)GC_register_dynamic_libraries(); #else GC_no_dls=TRUE; #endif } STATIC void GC_push_regs_and_stack(ptr_t cold_gc_frame) { #ifdef THREADS if (NULL==cold_gc_frame) return; #endif GC_with_callee_saves_pushed(GC_push_current_stack,cold_gc_frame); } GC_INNER void GC_push_roots(GC_bool all,ptr_t cold_gc_frame GC_ATTR_UNUSED) { int i; unsigned kind; #if!defined(REGISTER_LIBRARIES_EARLY) GC_cond_register_dynamic_libraries(); #endif for (i=0;i < n_root_sets;i++){ GC_push_conditional_with_exclusions( GC_static_roots[i].r_start, GC_static_roots[i].r_end,all); } for (kind=0;kind < GC_n_kinds;kind++){ void*base=GC_base(GC_obj_kinds[kind].ok_freelist); if (base!=NULL){ GC_set_mark_bit(base); } } #ifndef GC_NO_FINALIZATION GC_push_finalizer_structures(); #endif #ifdef THREADS if (GC_no_dls||GC_roots_were_cleared) GC_push_thread_structures(); #endif if (GC_push_typed_structures) GC_push_typed_structures(); #if defined(THREAD_LOCAL_ALLOC) if (GC_world_stopped) GC_mark_thread_local_free_lists(); #endif #ifndef STACK_NOT_SCANNED GC_push_regs_and_stack(cold_gc_frame); #endif if (GC_push_other_roots!=0){ (*GC_push_other_roots)(); } } #ifdef ENABLE_DISCLAIM #endif #include GC_INNER signed_word GC_bytes_found=0; #if defined(PARALLEL_MARK) GC_INNER signed_word GC_fl_builder_count=0; #endif #ifndef MAX_LEAKED #define MAX_LEAKED 40 #endif STATIC ptr_t GC_leaked[MAX_LEAKED]={ NULL }; STATIC unsigned GC_n_leaked=0; GC_INNER GC_bool GC_have_errors=FALSE; #if!defined(EAGER_SWEEP)&&defined(ENABLE_DISCLAIM) STATIC void GC_reclaim_unconditionally_marked(void); #endif GC_INLINE void GC_add_leaked(ptr_t leaked) { #ifndef SHORT_DBG_HDRS if (GC_findleak_delay_free&&!GC_check_leaked(leaked)) return; #endif GC_have_errors=TRUE; if (GC_n_leaked < MAX_LEAKED){ GC_leaked[GC_n_leaked++]=leaked; GC_set_mark_bit(leaked); } } GC_INNER void GC_print_all_errors(void) { static GC_bool printing_errors=FALSE; GC_bool have_errors; unsigned i,n_leaked; ptr_t leaked[MAX_LEAKED]; DCL_LOCK_STATE; LOCK(); if (printing_errors){ UNLOCK(); return; } have_errors=GC_have_errors; printing_errors=TRUE; n_leaked=GC_n_leaked; if (n_leaked > 0){ GC_ASSERT(n_leaked<=MAX_LEAKED); BCOPY(GC_leaked,leaked,n_leaked*sizeof(ptr_t)); GC_n_leaked=0; BZERO(GC_leaked,n_leaked*sizeof(ptr_t)); } UNLOCK(); if (GC_debugging_started){ GC_print_all_smashed(); } else { have_errors=FALSE; } if (n_leaked > 0){ GC_err_printf("Found %u leaked objects:\n",n_leaked); have_errors=TRUE; } for (i=0;i < n_leaked;i++){ ptr_t p=leaked[i]; #ifndef SKIP_LEAKED_OBJECTS_PRINTING GC_print_heap_obj(p); #endif GC_free(p); } if (have_errors #ifndef GC_ABORT_ON_LEAK &&GETENV("GC_ABORT_ON_LEAK")!=NULL #endif ){ ABORT("Leaked or smashed objects encountered"); } LOCK(); printing_errors=FALSE; UNLOCK(); } GC_INNER GC_bool GC_block_empty(hdr*hhdr) { return (hhdr->hb_n_marks==0); } STATIC GC_bool GC_block_nearly_full(hdr*hhdr,word sz) { return hhdr->hb_n_marks > HBLK_OBJS(sz)*7/8; } STATIC ptr_t GC_reclaim_clear(struct hblk*hbp,hdr*hhdr,word sz, ptr_t list,signed_word*count) { word bit_no=0; word*p,*q,*plim; signed_word n_bytes_found=0; GC_ASSERT(hhdr==GC_find_header((ptr_t)hbp)); #ifndef THREADS GC_ASSERT(sz==hhdr->hb_sz); #else #endif GC_ASSERT((sz&(BYTES_PER_WORD-1))==0); p=(word*)(hbp->hb_body); plim=(word*)(hbp->hb_body+HBLKSIZE - sz); while ((word)p<=(word)plim){ if (mark_bit_from_hdr(hhdr,bit_no)){ p=(word*)((ptr_t)p+sz); } else { n_bytes_found+=sz; obj_link(p)=list; list=((ptr_t)p); q=(word*)((ptr_t)p+sz); #ifdef USE_MARK_BYTES GC_ASSERT(!(sz&1) &&!((word)p&(2*sizeof(word)- 1))); p[1]=0; p+=2; while ((word)p < (word)q){ CLEAR_DOUBLE(p); p+=2; } #else p++; while ((word)p < (word)q){ *p++=0; } #endif } bit_no+=MARK_BIT_OFFSET(sz); } *count+=n_bytes_found; return(list); } STATIC ptr_t GC_reclaim_uninit(struct hblk*hbp,hdr*hhdr,word sz, ptr_t list,signed_word*count) { word bit_no=0; word*p,*plim; signed_word n_bytes_found=0; #ifndef THREADS GC_ASSERT(sz==hhdr->hb_sz); #endif p=(word*)(hbp->hb_body); plim=(word*)((ptr_t)hbp+HBLKSIZE - sz); while ((word)p<=(word)plim){ if (!mark_bit_from_hdr(hhdr,bit_no)){ n_bytes_found+=sz; obj_link(p)=list; list=((ptr_t)p); } p=(word*)((ptr_t)p+sz); bit_no+=MARK_BIT_OFFSET(sz); } *count+=n_bytes_found; return(list); } #ifdef ENABLE_DISCLAIM STATIC ptr_t GC_disclaim_and_reclaim(struct hblk*hbp,hdr*hhdr,word sz, ptr_t list,signed_word*count) { word bit_no=0; word*p,*q,*plim; signed_word n_bytes_found=0; struct obj_kind*ok=&GC_obj_kinds[hhdr->hb_obj_kind]; int (GC_CALLBACK*disclaim)(void*)=ok->ok_disclaim_proc; #ifndef THREADS GC_ASSERT(sz==hhdr->hb_sz); #endif p=(word*)(hbp->hb_body); plim=(word*)((ptr_t)p+HBLKSIZE - sz); while ((word)p<=(word)plim){ int marked=mark_bit_from_hdr(hhdr,bit_no); if (!marked&&(*disclaim)(p)){ set_mark_bit_from_hdr(hhdr,bit_no); hhdr->hb_n_marks++; marked=1; } if (marked) p=(word*)((ptr_t)p+sz); else { n_bytes_found+=sz; obj_link(p)=list; list=((ptr_t)p); q=(word*)((ptr_t)p+sz); #ifdef USE_MARK_BYTES GC_ASSERT((sz&1)==0); GC_ASSERT(((word)p&(2*sizeof(word)- 1))==0); p[1]=0; p+=2; while ((word)p < (word)q){ CLEAR_DOUBLE(p); p+=2; } #else p++; while ((word)p < (word)q){ *p++=0; } #endif } bit_no+=MARK_BIT_OFFSET(sz); } *count+=n_bytes_found; return list; } #endif STATIC void GC_reclaim_check(struct hblk*hbp,hdr*hhdr,word sz) { word bit_no; ptr_t p,plim; #ifndef THREADS GC_ASSERT(sz==hhdr->hb_sz); #endif p=hbp->hb_body; plim=p+HBLKSIZE - sz; for (bit_no=0;(word)p<=(word)plim; p+=sz,bit_no+=MARK_BIT_OFFSET(sz)){ if (!mark_bit_from_hdr(hhdr,bit_no)){ GC_add_leaked(p); } } } #ifdef AO_HAVE_load #define IS_PTRFREE_SAFE(hhdr)(AO_load((volatile AO_t*)&(hhdr)->hb_descr)==0) #else #define IS_PTRFREE_SAFE(hhdr)((hhdr)->hb_descr==0) #endif GC_INNER ptr_t GC_reclaim_generic(struct hblk*hbp,hdr*hhdr,size_t sz, GC_bool init,ptr_t list, signed_word*count) { ptr_t result; GC_ASSERT(GC_find_header((ptr_t)hbp)==hhdr); #ifndef GC_DISABLE_INCREMENTAL GC_remove_protection(hbp,1,IS_PTRFREE_SAFE(hhdr)); #endif #ifdef ENABLE_DISCLAIM if ((hhdr->hb_flags&HAS_DISCLAIM)!=0){ result=GC_disclaim_and_reclaim(hbp,hhdr,sz,list,count); } else #endif if (init||GC_debugging_started){ result=GC_reclaim_clear(hbp,hhdr,sz,list,count); } else { GC_ASSERT(IS_PTRFREE_SAFE(hhdr)); result=GC_reclaim_uninit(hbp,hhdr,sz,list,count); } if (IS_UNCOLLECTABLE(hhdr->hb_obj_kind))GC_set_hdr_marks(hhdr); return result; } STATIC void GC_reclaim_small_nonempty_block(struct hblk*hbp,word sz, GC_bool report_if_found) { hdr*hhdr=HDR(hbp); struct obj_kind*ok=&GC_obj_kinds[hhdr->hb_obj_kind]; void**flh=&(ok->ok_freelist[BYTES_TO_GRANULES(sz)]); hhdr->hb_last_reclaimed=(unsigned short)GC_gc_no; if (report_if_found){ GC_reclaim_check(hbp,hhdr,sz); } else { *flh=GC_reclaim_generic(hbp,hhdr,sz,ok->ok_init, (ptr_t)(*flh),&GC_bytes_found); } } #ifdef ENABLE_DISCLAIM STATIC void GC_disclaim_and_reclaim_or_free_small_block(struct hblk*hbp) { hdr*hhdr=HDR(hbp); word sz=hhdr->hb_sz; struct obj_kind*ok=&GC_obj_kinds[hhdr->hb_obj_kind]; void**flh=&(ok->ok_freelist[BYTES_TO_GRANULES(sz)]); void*flh_next; hhdr->hb_last_reclaimed=(unsigned short)GC_gc_no; flh_next=GC_reclaim_generic(hbp,hhdr,sz,ok->ok_init, (ptr_t)(*flh),&GC_bytes_found); if (hhdr->hb_n_marks) *flh=flh_next; else { GC_bytes_found+=HBLKSIZE; GC_freehblk(hbp); } } #endif STATIC void GC_reclaim_block(struct hblk*hbp,word report_if_found) { hdr*hhdr=HDR(hbp); word sz; struct obj_kind*ok=&GC_obj_kinds[hhdr->hb_obj_kind]; #ifdef AO_HAVE_load sz=(word)AO_load((volatile AO_t*)&hhdr->hb_sz); #else sz=hhdr->hb_sz; #endif if( sz > MAXOBJBYTES){ if(!mark_bit_from_hdr(hhdr,0)){ if (report_if_found){ GC_add_leaked((ptr_t)hbp); } else { word blocks; #ifdef ENABLE_DISCLAIM if (EXPECT(hhdr->hb_flags&HAS_DISCLAIM,0)){ if ((*ok->ok_disclaim_proc)(hbp)){ set_mark_bit_from_hdr(hhdr,0); goto in_use; } } #endif blocks=OBJ_SZ_TO_BLOCKS(sz); #if defined(CPPCHECK) GC_noop1((word)&blocks); #endif if (blocks > 1){ GC_large_allocd_bytes-=blocks*HBLKSIZE; } GC_bytes_found+=sz; GC_freehblk(hbp); } } else { #ifdef ENABLE_DISCLAIM in_use: #endif if (IS_PTRFREE_SAFE(hhdr)){ GC_atomic_in_use+=sz; } else { GC_composite_in_use+=sz; } } } else { GC_bool empty=GC_block_empty(hhdr); #ifdef PARALLEL_MARK GC_ASSERT(hhdr->hb_n_marks<=2*(HBLKSIZE/sz+1)+16); #else GC_ASSERT(sz*hhdr->hb_n_marks<=HBLKSIZE); #endif if (report_if_found){ GC_reclaim_small_nonempty_block(hbp,sz, TRUE); } else if (empty){ #ifdef ENABLE_DISCLAIM if ((hhdr->hb_flags&HAS_DISCLAIM)!=0){ GC_disclaim_and_reclaim_or_free_small_block(hbp); } else #endif { GC_bytes_found+=HBLKSIZE; GC_freehblk(hbp); } } else if (GC_find_leak||!GC_block_nearly_full(hhdr,sz)){ struct hblk**rlh=ok->ok_reclaim_list; if (rlh!=NULL){ rlh+=BYTES_TO_GRANULES(sz); hhdr->hb_next=*rlh; *rlh=hbp; } } if (IS_PTRFREE_SAFE(hhdr)){ GC_atomic_in_use+=sz*hhdr->hb_n_marks; } else { GC_composite_in_use+=sz*hhdr->hb_n_marks; } } } #if!defined(NO_DEBUGGING) struct Print_stats { size_t number_of_blocks; size_t total_bytes; }; #ifdef USE_MARK_BYTES unsigned GC_n_set_marks(hdr*hhdr) { unsigned result=0; word i; word sz=hhdr->hb_sz; word offset=MARK_BIT_OFFSET(sz); word limit=FINAL_MARK_BIT(sz); for (i=0;i < limit;i+=offset){ result+=hhdr->hb_marks[i]; } GC_ASSERT(hhdr->hb_marks[limit]); return(result); } #else static unsigned set_bits(word n) { word m=n; unsigned result=0; while (m > 0){ if (m&1)result++; m>>=1; } return(result); } unsigned GC_n_set_marks(hdr*hhdr) { unsigned result=0; word i; word n_mark_words; #ifdef MARK_BIT_PER_OBJ word n_objs=HBLK_OBJS(hhdr->hb_sz); if (0==n_objs)n_objs=1; n_mark_words=divWORDSZ(n_objs+WORDSZ - 1); #else n_mark_words=MARK_BITS_SZ; #endif for (i=0;i < n_mark_words - 1;i++){ result+=set_bits(hhdr->hb_marks[i]); } #ifdef MARK_BIT_PER_OBJ result+=set_bits((hhdr->hb_marks[n_mark_words - 1]) <<(n_mark_words*WORDSZ - n_objs)); #else result+=set_bits(hhdr->hb_marks[n_mark_words - 1]); #endif return result; } #endif STATIC void GC_print_block_descr(struct hblk*h, word raw_ps) { hdr*hhdr=HDR(h); size_t bytes=hhdr->hb_sz; struct Print_stats*ps; unsigned n_marks=GC_n_set_marks(hhdr); unsigned n_objs=(unsigned)HBLK_OBJS(bytes); if (0==n_objs)n_objs=1; if (hhdr->hb_n_marks!=n_marks){ GC_printf("%u,%u,%u!=%u,%u\n",hhdr->hb_obj_kind,(unsigned)bytes, (unsigned)hhdr->hb_n_marks,n_marks,n_objs); } else { GC_printf("%u,%u,%u,%u\n",hhdr->hb_obj_kind,(unsigned)bytes, n_marks,n_objs); } ps=(struct Print_stats*)raw_ps; ps->total_bytes+=(bytes+(HBLKSIZE-1))&~(HBLKSIZE-1); ps->number_of_blocks++; } void GC_print_block_list(void) { struct Print_stats pstats; GC_printf("kind(0=ptrfree,1=normal,2=unc.)," "size_in_bytes,#_marks_set,#objs\n"); pstats.number_of_blocks=0; pstats.total_bytes=0; GC_apply_to_all_blocks(GC_print_block_descr,(word)&pstats); GC_printf("blocks=%lu,bytes=%lu\n", (unsigned long)pstats.number_of_blocks, (unsigned long)pstats.total_bytes); } GC_API void GC_CALL GC_print_free_list(int kind,size_t sz_in_granules) { void*flh_next; int n; GC_ASSERT(kind < MAXOBJKINDS); GC_ASSERT(sz_in_granules<=MAXOBJGRANULES); flh_next=GC_obj_kinds[kind].ok_freelist[sz_in_granules]; for (n=0;flh_next;n++){ GC_printf("Free object in heap block %p [%d]:%p\n", (void*)HBLKPTR(flh_next),n,flh_next); flh_next=obj_link(flh_next); } } #endif STATIC void GC_clear_fl_links(void**flp) { void*next=*flp; while (0!=next){ *flp=0; flp=&(obj_link(next)); next=*flp; } } GC_INNER void GC_start_reclaim(GC_bool report_if_found) { unsigned kind; #if defined(PARALLEL_MARK) GC_ASSERT(0==GC_fl_builder_count); #endif GC_composite_in_use=0; GC_atomic_in_use=0; for (kind=0;kind < GC_n_kinds;kind++){ struct hblk**rlist=GC_obj_kinds[kind].ok_reclaim_list; GC_bool should_clobber=(GC_obj_kinds[kind].ok_descriptor!=0); if (rlist==0)continue; if (!report_if_found){ void**fop; void**lim=&(GC_obj_kinds[kind].ok_freelist[MAXOBJGRANULES+1]); for (fop=GC_obj_kinds[kind].ok_freelist; (word)fop < (word)lim;(*(word**)&fop)++){ if (*fop!=0){ if (should_clobber){ GC_clear_fl_links(fop); } else { *fop=0; } } } } BZERO(rlist,(MAXOBJGRANULES+1)*sizeof(void*)); } GC_apply_to_all_blocks(GC_reclaim_block,(word)report_if_found); #ifdef EAGER_SWEEP GC_reclaim_all((GC_stop_func)0,FALSE); #elif defined(ENABLE_DISCLAIM) GC_reclaim_unconditionally_marked(); #endif #if defined(PARALLEL_MARK) GC_ASSERT(0==GC_fl_builder_count); #endif } GC_INNER void GC_continue_reclaim(word sz,int kind) { hdr*hhdr; struct hblk*hbp; struct obj_kind*ok=&(GC_obj_kinds[kind]); struct hblk**rlh=ok->ok_reclaim_list; void**flh=&(ok->ok_freelist[sz]); if (NULL==rlh) return; for (rlh+=sz;(hbp=*rlh)!=NULL;){ hhdr=HDR(hbp); *rlh=hhdr->hb_next; GC_reclaim_small_nonempty_block(hbp,hhdr->hb_sz,FALSE); if (*flh!=0) break; } } GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func,GC_bool ignore_old) { word sz; unsigned kind; hdr*hhdr; struct hblk*hbp; struct obj_kind*ok; struct hblk**rlp; struct hblk**rlh; #ifndef NO_CLOCK CLOCK_TYPE start_time=CLOCK_TYPE_INITIALIZER; if (GC_print_stats==VERBOSE) GET_TIME(start_time); #endif for (kind=0;kind < GC_n_kinds;kind++){ ok=&(GC_obj_kinds[kind]); rlp=ok->ok_reclaim_list; if (rlp==0)continue; for (sz=1;sz<=MAXOBJGRANULES;sz++){ for (rlh=rlp+sz;(hbp=*rlh)!=NULL;){ if (stop_func!=(GC_stop_func)0&&(*stop_func)()){ return(FALSE); } hhdr=HDR(hbp); *rlh=hhdr->hb_next; if (!ignore_old ||(word)hhdr->hb_last_reclaimed==GC_gc_no - 1){ GC_reclaim_small_nonempty_block(hbp,hhdr->hb_sz,FALSE); } } } } #ifndef NO_CLOCK if (GC_print_stats==VERBOSE){ CLOCK_TYPE done_time; GET_TIME(done_time); GC_verbose_log_printf( "Disposing of reclaim lists took %lu ms %lu ns\n", MS_TIME_DIFF(done_time,start_time), NS_FRAC_TIME_DIFF(done_time,start_time)); } #endif return(TRUE); } #if!defined(EAGER_SWEEP)&&defined(ENABLE_DISCLAIM) STATIC void GC_reclaim_unconditionally_marked(void) { word sz; unsigned kind; hdr*hhdr; struct hblk*hbp; struct obj_kind*ok; struct hblk**rlp; struct hblk**rlh; for (kind=0;kind < GC_n_kinds;kind++){ ok=&(GC_obj_kinds[kind]); if (!ok->ok_mark_unconditionally) continue; rlp=ok->ok_reclaim_list; if (rlp==0) continue; for (sz=1;sz<=MAXOBJGRANULES;sz++){ rlh=rlp+sz; while ((hbp=*rlh)!=0){ hhdr=HDR(hbp); *rlh=hhdr->hb_next; GC_reclaim_small_nonempty_block(hbp,hhdr->hb_sz,FALSE); } } } } #endif struct enumerate_reachable_s { GC_reachable_object_proc proc; void*client_data; }; STATIC void GC_do_enumerate_reachable_objects(struct hblk*hbp,word ped) { struct hblkhdr*hhdr=HDR(hbp); size_t sz=(size_t)hhdr->hb_sz; size_t bit_no; char*p,*plim; if (GC_block_empty(hhdr)){ return; } p=hbp->hb_body; if (sz > MAXOBJBYTES){ plim=p; } else { plim=hbp->hb_body+HBLKSIZE - sz; } for (bit_no=0;p<=plim;bit_no+=MARK_BIT_OFFSET(sz),p+=sz){ if (mark_bit_from_hdr(hhdr,bit_no)){ ((struct enumerate_reachable_s*)ped)->proc(p,sz, ((struct enumerate_reachable_s*)ped)->client_data); } } } GC_API void GC_CALL GC_enumerate_reachable_objects_inner( GC_reachable_object_proc proc, void*client_data) { struct enumerate_reachable_s ed; GC_ASSERT(I_HOLD_LOCK()); ed.proc=proc; ed.client_data=client_data; GC_apply_to_all_blocks(GC_do_enumerate_reachable_objects,(word)&ed); } #ifndef GC_TYPED_H #define GC_TYPED_H #ifndef GC_H #endif #ifdef __cplusplus extern "C" { #endif typedef GC_word*GC_bitmap; #define GC_WORDSZ (8*sizeof(GC_word)) #define GC_get_bit(bm,index)(((bm)[(index)/GC_WORDSZ]>>((index)% GC_WORDSZ))&1) #define GC_set_bit(bm,index)((bm)[(index)/GC_WORDSZ]|=(GC_word)1<<((index)% GC_WORDSZ)) #define GC_WORD_OFFSET(t,f)(offsetof(t,f)/sizeof(GC_word)) #define GC_WORD_LEN(t)(sizeof(t)/sizeof(GC_word)) #define GC_BITMAP_SIZE(t)((GC_WORD_LEN(t)+GC_WORDSZ - 1)/GC_WORDSZ) typedef GC_word GC_descr; GC_API GC_descr GC_CALL GC_make_descriptor(const GC_word*, size_t); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_malloc_explicitly_typed(size_t, GC_descr); GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1)void*GC_CALL GC_malloc_explicitly_typed_ignore_off_page(size_t, GC_descr); GC_API GC_ATTR_MALLOC GC_ATTR_CALLOC_SIZE(1,2)void*GC_CALL GC_calloc_explicitly_typed(size_t, size_t, GC_descr); #ifdef GC_DEBUG #define GC_MALLOC_EXPLICITLY_TYPED(bytes,d)((void)(d),GC_MALLOC(bytes)) #define GC_CALLOC_EXPLICITLY_TYPED(n,bytes,d)((void)(d),GC_MALLOC((n)*(bytes))) #else #define GC_MALLOC_EXPLICITLY_TYPED(bytes,d)GC_malloc_explicitly_typed(bytes,d) #define GC_CALLOC_EXPLICITLY_TYPED(n,bytes,d)GC_calloc_explicitly_typed(n,bytes,d) #endif #ifdef __cplusplus } #endif #endif #define TYPD_EXTRA_BYTES (sizeof(word)- EXTRA_BYTES) STATIC int GC_explicit_kind=0; STATIC int GC_array_kind=0; struct LeafDescriptor { word ld_tag; #define LEAF_TAG 1 size_t ld_size; size_t ld_nelements; GC_descr ld_descriptor; }; struct ComplexArrayDescriptor { word ad_tag; #define ARRAY_TAG 2 size_t ad_nelements; union ComplexDescriptor*ad_element_descr; }; struct SequenceDescriptor { word sd_tag; #define SEQUENCE_TAG 3 union ComplexDescriptor*sd_first; union ComplexDescriptor*sd_second; }; typedef union ComplexDescriptor { struct LeafDescriptor ld; struct ComplexArrayDescriptor ad; struct SequenceDescriptor sd; } complex_descriptor; #define TAG ad.ad_tag #define ED_INITIAL_SIZE 100 STATIC int GC_typed_mark_proc_index=0; STATIC int GC_array_mark_proc_index=0; STATIC void GC_push_typed_structures_proc(void) { GC_PUSH_ALL_SYM(GC_ext_descriptors); } STATIC signed_word GC_add_ext_descriptor(const word*bm,word nbits) { size_t nwords=divWORDSZ(nbits+WORDSZ-1); signed_word result; size_t i; word last_part; size_t extra_bits; DCL_LOCK_STATE; LOCK(); while (GC_avail_descr+nwords>=GC_ed_size){ typed_ext_descr_t*newExtD; size_t new_size; word ed_size=GC_ed_size; if (ed_size==0){ GC_ASSERT((word)(&GC_ext_descriptors)% sizeof(word)==0); GC_push_typed_structures=GC_push_typed_structures_proc; UNLOCK(); new_size=ED_INITIAL_SIZE; } else { UNLOCK(); new_size=2*ed_size; if (new_size > MAX_ENV)return(-1); } newExtD=(typed_ext_descr_t*)GC_malloc_atomic(new_size *sizeof(typed_ext_descr_t)); if (NULL==newExtD) return -1; LOCK(); if (ed_size==GC_ed_size){ if (GC_avail_descr!=0){ BCOPY(GC_ext_descriptors,newExtD, GC_avail_descr*sizeof(typed_ext_descr_t)); } GC_ed_size=new_size; GC_ext_descriptors=newExtD; } } result=GC_avail_descr; for (i=0;i < nwords-1;i++){ GC_ext_descriptors[result+i].ed_bitmap=bm[i]; GC_ext_descriptors[result+i].ed_continued=TRUE; } last_part=bm[i]; extra_bits=nwords*WORDSZ - nbits; last_part<<=extra_bits; last_part>>=extra_bits; GC_ext_descriptors[result+i].ed_bitmap=last_part; GC_ext_descriptors[result+i].ed_continued=FALSE; GC_avail_descr+=nwords; UNLOCK(); return(result); } STATIC GC_descr GC_bm_table[WORDSZ/2]; STATIC GC_descr GC_double_descr(GC_descr descriptor,word nwords) { if ((descriptor&GC_DS_TAGS)==GC_DS_LENGTH){ descriptor=GC_bm_table[BYTES_TO_WORDS((word)descriptor)]; }; descriptor|=(descriptor&~GC_DS_TAGS)>>nwords; return(descriptor); } STATIC complex_descriptor* GC_make_sequence_descriptor(complex_descriptor*first, complex_descriptor*second); #define COMPLEX 2 #define LEAF 1 #define SIMPLE 0 #define NO_MEM (-1) STATIC int GC_make_array_descriptor(size_t nelements,size_t size, GC_descr descriptor,GC_descr*simple_d, complex_descriptor**complex_d, struct LeafDescriptor*leaf) { #define OPT_THRESHOLD 50 if ((descriptor&GC_DS_TAGS)==GC_DS_LENGTH){ if (descriptor==(GC_descr)size){ *simple_d=nelements*descriptor; return(SIMPLE); } else if ((word)descriptor==0){ *simple_d=(GC_descr)0; return(SIMPLE); } } if (nelements<=OPT_THRESHOLD){ if (nelements<=1){ if (nelements==1){ *simple_d=descriptor; return(SIMPLE); } else { *simple_d=(GC_descr)0; return(SIMPLE); } } } else if (size<=BITMAP_BITS/2 &&(descriptor&GC_DS_TAGS)!=GC_DS_PROC &&(size&(sizeof(word)-1))==0){ int result= GC_make_array_descriptor(nelements/2,2*size, GC_double_descr(descriptor, BYTES_TO_WORDS(size)), simple_d,complex_d,leaf); if ((nelements&1)==0){ return(result); } else { struct LeafDescriptor*one_element= (struct LeafDescriptor*) GC_malloc_atomic(sizeof(struct LeafDescriptor)); if (result==NO_MEM||one_element==0)return(NO_MEM); one_element->ld_tag=LEAF_TAG; one_element->ld_size=size; one_element->ld_nelements=1; one_element->ld_descriptor=descriptor; switch(result){ case SIMPLE: { struct LeafDescriptor*beginning= (struct LeafDescriptor*) GC_malloc_atomic(sizeof(struct LeafDescriptor)); if (beginning==0)return(NO_MEM); beginning->ld_tag=LEAF_TAG; beginning->ld_size=size; beginning->ld_nelements=1; beginning->ld_descriptor=*simple_d; *complex_d=GC_make_sequence_descriptor( (complex_descriptor*)beginning, (complex_descriptor*)one_element); break; } case LEAF: { struct LeafDescriptor*beginning= (struct LeafDescriptor*) GC_malloc_atomic(sizeof(struct LeafDescriptor)); if (beginning==0)return(NO_MEM); beginning->ld_tag=LEAF_TAG; beginning->ld_size=leaf->ld_size; beginning->ld_nelements=leaf->ld_nelements; beginning->ld_descriptor=leaf->ld_descriptor; *complex_d=GC_make_sequence_descriptor( (complex_descriptor*)beginning, (complex_descriptor*)one_element); break; } case COMPLEX: *complex_d=GC_make_sequence_descriptor( *complex_d, (complex_descriptor*)one_element); break; } return(COMPLEX); } } leaf->ld_size=size; leaf->ld_nelements=nelements; leaf->ld_descriptor=descriptor; return(LEAF); } STATIC complex_descriptor* GC_make_sequence_descriptor(complex_descriptor*first, complex_descriptor*second) { struct SequenceDescriptor*result= (struct SequenceDescriptor*) GC_malloc(sizeof(struct SequenceDescriptor)); if (result!=0){ result->sd_tag=SEQUENCE_TAG; result->sd_first=first; result->sd_second=second; GC_dirty(result); REACHABLE_AFTER_DIRTY(first); REACHABLE_AFTER_DIRTY(second); } return((complex_descriptor*)result); } STATIC mse*GC_typed_mark_proc(word*addr,mse*mark_stack_ptr, mse*mark_stack_limit,word env); STATIC mse*GC_array_mark_proc(word*addr,mse*mark_stack_ptr, mse*mark_stack_limit,word env); STATIC void GC_init_explicit_typing(void) { unsigned i; GC_STATIC_ASSERT(sizeof(struct LeafDescriptor)% sizeof(word)==0); GC_explicit_kind=GC_new_kind_inner(GC_new_free_list_inner(), (WORDS_TO_BYTES((word)-1)|GC_DS_PER_OBJECT), TRUE,TRUE); GC_typed_mark_proc_index=GC_new_proc_inner(GC_typed_mark_proc); GC_array_mark_proc_index=GC_new_proc_inner(GC_array_mark_proc); GC_array_kind=GC_new_kind_inner(GC_new_free_list_inner(), GC_MAKE_PROC(GC_array_mark_proc_index,0), FALSE,TRUE); GC_bm_table[0]=GC_DS_BITMAP; for (i=1;i < WORDSZ/2;i++){ GC_bm_table[i]=(((word)-1)<<(WORDSZ - i))|GC_DS_BITMAP; } } STATIC mse*GC_typed_mark_proc(word*addr,mse*mark_stack_ptr, mse*mark_stack_limit,word env) { word bm=GC_ext_descriptors[env].ed_bitmap; word*current_p=addr; word current; ptr_t greatest_ha=(ptr_t)GC_greatest_plausible_heap_addr; ptr_t least_ha=(ptr_t)GC_least_plausible_heap_addr; DECLARE_HDR_CACHE; INIT_HDR_CACHE; for (;bm!=0;bm>>=1,current_p++){ if (bm&1){ current=*current_p; FIXUP_POINTER(current); if (current>=(word)least_ha&¤t<=(word)greatest_ha){ PUSH_CONTENTS((ptr_t)current,mark_stack_ptr, mark_stack_limit,(ptr_t)current_p); } } } if (GC_ext_descriptors[env].ed_continued){ mark_stack_ptr++; if ((word)mark_stack_ptr>=(word)mark_stack_limit){ mark_stack_ptr=GC_signal_mark_stack_overflow(mark_stack_ptr); } mark_stack_ptr->mse_start=(ptr_t)(addr+WORDSZ); mark_stack_ptr->mse_descr.w= GC_MAKE_PROC(GC_typed_mark_proc_index,env+1); } return(mark_stack_ptr); } STATIC word GC_descr_obj_size(complex_descriptor*d) { switch(d->TAG){ case LEAF_TAG: return(d->ld.ld_nelements*d->ld.ld_size); case ARRAY_TAG: return(d->ad.ad_nelements *GC_descr_obj_size(d->ad.ad_element_descr)); case SEQUENCE_TAG: return(GC_descr_obj_size(d->sd.sd_first) +GC_descr_obj_size(d->sd.sd_second)); default: ABORT_RET("Bad complex descriptor"); return 0; } } STATIC mse*GC_push_complex_descriptor(word*addr,complex_descriptor*d, mse*msp,mse*msl) { ptr_t current=(ptr_t)addr; word nelements; word sz; word i; switch(d->TAG){ case LEAF_TAG: { GC_descr descr=d->ld.ld_descriptor; nelements=d->ld.ld_nelements; if (msl - msp<=(ptrdiff_t)nelements)return(0); sz=d->ld.ld_size; for (i=0;i < nelements;i++){ msp++; msp->mse_start=current; msp->mse_descr.w=descr; current+=sz; } return(msp); } case ARRAY_TAG: { complex_descriptor*descr=d->ad.ad_element_descr; nelements=d->ad.ad_nelements; sz=GC_descr_obj_size(descr); for (i=0;i < nelements;i++){ msp=GC_push_complex_descriptor((word*)current,descr, msp,msl); if (msp==0)return(0); current+=sz; } return(msp); } case SEQUENCE_TAG: { sz=GC_descr_obj_size(d->sd.sd_first); msp=GC_push_complex_descriptor((word*)current,d->sd.sd_first, msp,msl); if (msp==0)return(0); current+=sz; msp=GC_push_complex_descriptor((word*)current,d->sd.sd_second, msp,msl); return(msp); } default: ABORT_RET("Bad complex descriptor"); return 0; } } STATIC mse*GC_array_mark_proc(word*addr,mse*mark_stack_ptr, mse*mark_stack_limit, word env GC_ATTR_UNUSED) { hdr*hhdr=HDR(addr); word sz=hhdr->hb_sz; word nwords=BYTES_TO_WORDS(sz); complex_descriptor*descr=(complex_descriptor*)(addr[nwords-1]); mse*orig_mark_stack_ptr=mark_stack_ptr; mse*new_mark_stack_ptr; if (descr==0){ return(orig_mark_stack_ptr); } new_mark_stack_ptr=GC_push_complex_descriptor(addr,descr, mark_stack_ptr, mark_stack_limit-1); if (new_mark_stack_ptr==0){ if (NULL==mark_stack_ptr)ABORT("Bad mark_stack_ptr"); #ifdef PARALLEL_MARK if (GC_mark_stack+GC_mark_stack_size==mark_stack_limit) #endif { GC_mark_stack_too_small=TRUE; } new_mark_stack_ptr=orig_mark_stack_ptr+1; new_mark_stack_ptr->mse_start=(ptr_t)addr; new_mark_stack_ptr->mse_descr.w=sz|GC_DS_LENGTH; } else { new_mark_stack_ptr++; new_mark_stack_ptr->mse_start=(ptr_t)(addr+nwords - 1); new_mark_stack_ptr->mse_descr.w=sizeof(word)|GC_DS_LENGTH; } return new_mark_stack_ptr; } GC_API GC_descr GC_CALL GC_make_descriptor(const GC_word*bm,size_t len) { signed_word last_set_bit=len - 1; GC_descr result; DCL_LOCK_STATE; #if defined(AO_HAVE_load_acquire)&&defined(AO_HAVE_store_release) if (!EXPECT(AO_load_acquire(&GC_explicit_typing_initialized),TRUE)){ LOCK(); if (!GC_explicit_typing_initialized){ GC_init_explicit_typing(); AO_store_release(&GC_explicit_typing_initialized,TRUE); } UNLOCK(); } #else LOCK(); if (!EXPECT(GC_explicit_typing_initialized,TRUE)){ GC_init_explicit_typing(); GC_explicit_typing_initialized=TRUE; } UNLOCK(); #endif while (last_set_bit>=0&&!GC_get_bit(bm,last_set_bit)) last_set_bit--; if (last_set_bit < 0)return(0); #if ALIGNMENT==CPP_WORDSZ/8 { signed_word i; for (i=0;i < last_set_bit;i++){ if (!GC_get_bit(bm,i)){ break; } } if (i==last_set_bit){ return (WORDS_TO_BYTES(last_set_bit+1)|GC_DS_LENGTH); } } #endif if ((word)last_set_bit < BITMAP_BITS){ signed_word i; result=SIGNB; for (i=last_set_bit - 1;i>=0;i--){ result>>=1; if (GC_get_bit(bm,i))result|=SIGNB; } result|=GC_DS_BITMAP; } else { signed_word index=GC_add_ext_descriptor(bm,(word)last_set_bit+1); if (index==-1)return(WORDS_TO_BYTES(last_set_bit+1)|GC_DS_LENGTH); result=GC_MAKE_PROC(GC_typed_mark_proc_index,(word)index); } return result; } GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_explicitly_typed(size_t lb, GC_descr d) { word*op; size_t lg; GC_ASSERT(GC_explicit_typing_initialized); lb=SIZET_SAT_ADD(lb,TYPD_EXTRA_BYTES); op=(word*)GC_malloc_kind(lb,GC_explicit_kind); if (EXPECT(NULL==op,FALSE)) return NULL; lg=BYTES_TO_GRANULES(GC_size(op)); op[GRANULES_TO_WORDS(lg)- 1]=d; GC_dirty(op+GRANULES_TO_WORDS(lg)- 1); REACHABLE_AFTER_DIRTY(d); return op; } #define GENERAL_MALLOC_IOP(lb,k)GC_clear_stack(GC_generic_malloc_ignore_off_page(lb,k)) GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_explicitly_typed_ignore_off_page(size_t lb,GC_descr d) { ptr_t op; size_t lg; DCL_LOCK_STATE; GC_ASSERT(GC_explicit_typing_initialized); lb=SIZET_SAT_ADD(lb,TYPD_EXTRA_BYTES); if (SMALL_OBJ(lb)){ void**opp; GC_DBG_COLLECT_AT_MALLOC(lb); LOCK(); lg=GC_size_map[lb]; opp=&GC_obj_kinds[GC_explicit_kind].ok_freelist[lg]; op=(ptr_t)(*opp); if (EXPECT(0==op,FALSE)){ UNLOCK(); op=(ptr_t)GENERAL_MALLOC_IOP(lb,GC_explicit_kind); if (0==op)return 0; lg=BYTES_TO_GRANULES(GC_size(op)); } else { *opp=obj_link(op); obj_link(op)=0; GC_bytes_allocd+=GRANULES_TO_BYTES((word)lg); UNLOCK(); } } else { op=(ptr_t)GENERAL_MALLOC_IOP(lb,GC_explicit_kind); if (NULL==op)return NULL; lg=BYTES_TO_GRANULES(GC_size(op)); } ((word*)op)[GRANULES_TO_WORDS(lg)- 1]=d; GC_dirty(op+GRANULES_TO_WORDS(lg)- 1); REACHABLE_AFTER_DIRTY(d); return op; } GC_API GC_ATTR_MALLOC void*GC_CALL GC_calloc_explicitly_typed(size_t n, size_t lb,GC_descr d) { word*op; size_t lg; GC_descr simple_descr; complex_descriptor*complex_descr; int descr_type; struct LeafDescriptor leaf; GC_ASSERT(GC_explicit_typing_initialized); descr_type=GC_make_array_descriptor((word)n,(word)lb,d,&simple_descr, &complex_descr,&leaf); if ((lb|n)> GC_SQRT_SIZE_MAX &&lb > 0&&n > GC_SIZE_MAX/lb) return (*GC_get_oom_fn())(GC_SIZE_MAX); lb*=n; switch(descr_type){ case NO_MEM:return(0); case SIMPLE: return GC_malloc_explicitly_typed(lb,simple_descr); case LEAF: lb=SIZET_SAT_ADD(lb, sizeof(struct LeafDescriptor)+TYPD_EXTRA_BYTES); break; case COMPLEX: lb=SIZET_SAT_ADD(lb,TYPD_EXTRA_BYTES); break; } op=(word*)GC_malloc_kind(lb,GC_array_kind); if (EXPECT(NULL==op,FALSE)) return NULL; lg=BYTES_TO_GRANULES(GC_size(op)); if (descr_type==LEAF){ volatile struct LeafDescriptor*lp= (struct LeafDescriptor*) (op+GRANULES_TO_WORDS(lg) - (BYTES_TO_WORDS(sizeof(struct LeafDescriptor))+1)); lp->ld_tag=LEAF_TAG; lp->ld_size=leaf.ld_size; lp->ld_nelements=leaf.ld_nelements; lp->ld_descriptor=leaf.ld_descriptor; ((volatile word*)op)[GRANULES_TO_WORDS(lg)- 1]=(word)lp; } else { #ifndef GC_NO_FINALIZATION size_t lw=GRANULES_TO_WORDS(lg); op[lw - 1]=(word)complex_descr; GC_dirty(op+lw - 1); REACHABLE_AFTER_DIRTY(complex_descr); if (EXPECT(GC_general_register_disappearing_link( (void**)(op+lw - 1),op) ==GC_NO_MEMORY,FALSE)) #endif { return (*GC_get_oom_fn())(lb); } } return op; } #include #include #include #ifndef MSWINCE #include #endif #ifdef GC_SOLARIS_THREADS #include #endif #if defined(UNIX_LIKE)||defined(CYGWIN32)||defined(SYMBIAN)||(defined(CONSOLE_LOG)&&defined(MSWIN32)) #include #include #include #endif #if defined(CONSOLE_LOG)&&defined(MSWIN32)&&defined(_MSC_VER) #include #endif #ifdef NONSTOP #include #endif #ifdef THREADS #ifdef PCR #include "il/PCR_IL.h" GC_INNER PCR_Th_ML GC_allocate_ml; #elif defined(SN_TARGET_PSP2) GC_INNER WapiMutex GC_allocate_ml_PSP2={ 0,NULL }; #elif defined(SN_TARGET_ORBIS)||defined(SN_TARGET_PS3) #include GC_INNER pthread_mutex_t GC_allocate_ml; #endif #endif #ifdef DYNAMIC_LOADING #define GC_REGISTER_MAIN_STATIC_DATA()GC_register_main_static_data() #elif defined(GC_DONT_REGISTER_MAIN_STATIC_DATA) #define GC_REGISTER_MAIN_STATIC_DATA()FALSE #else #define GC_REGISTER_MAIN_STATIC_DATA()TRUE #endif #ifdef NEED_CANCEL_DISABLE_COUNT __thread unsigned char GC_cancel_disable_count=0; #endif GC_FAR struct _GC_arrays GC_arrays; GC_INNER unsigned GC_n_mark_procs=GC_RESERVED_MARK_PROCS; GC_INNER unsigned GC_n_kinds=GC_N_KINDS_INITIAL_VALUE; GC_INNER GC_bool GC_debugging_started=FALSE; ptr_t GC_stackbottom=0; #ifdef IA64 ptr_t GC_register_stackbottom=0; #endif int GC_dont_gc=FALSE; int GC_dont_precollect=FALSE; GC_bool GC_quiet=0; #if!defined(NO_CLOCK)||!defined(SMALL_CONFIG) int GC_print_stats=0; #endif #ifdef GC_PRINT_BACK_HEIGHT GC_INNER GC_bool GC_print_back_height=TRUE; #else GC_INNER GC_bool GC_print_back_height=FALSE; #endif #ifndef NO_DEBUGGING #ifdef GC_DUMP_REGULARLY GC_INNER GC_bool GC_dump_regularly=TRUE; #else GC_INNER GC_bool GC_dump_regularly=FALSE; #endif #ifndef NO_CLOCK STATIC CLOCK_TYPE GC_init_time; #endif #endif #ifdef KEEP_BACK_PTRS GC_INNER long GC_backtraces=0; #endif #ifdef FIND_LEAK int GC_find_leak=1; #else int GC_find_leak=0; #endif #ifndef SHORT_DBG_HDRS #ifdef GC_FINDLEAK_DELAY_FREE GC_INNER GC_bool GC_findleak_delay_free=TRUE; #else GC_INNER GC_bool GC_findleak_delay_free=FALSE; #endif #endif #ifdef ALL_INTERIOR_POINTERS int GC_all_interior_pointers=1; #else int GC_all_interior_pointers=0; #endif #ifdef FINALIZE_ON_DEMAND int GC_finalize_on_demand=1; #else int GC_finalize_on_demand=0; #endif #ifdef JAVA_FINALIZATION int GC_java_finalization=1; #else int GC_java_finalization=0; #endif GC_finalizer_notifier_proc GC_finalizer_notifier= (GC_finalizer_notifier_proc)0; #ifdef GC_FORCE_UNMAP_ON_GCOLLECT GC_INNER GC_bool GC_force_unmap_on_gcollect=TRUE; #else GC_INNER GC_bool GC_force_unmap_on_gcollect=FALSE; #endif #ifndef GC_LARGE_ALLOC_WARN_INTERVAL #define GC_LARGE_ALLOC_WARN_INTERVAL 5 #endif GC_INNER long GC_large_alloc_warn_interval=GC_LARGE_ALLOC_WARN_INTERVAL; STATIC void*GC_CALLBACK GC_default_oom_fn( size_t bytes_requested GC_ATTR_UNUSED) { return(0); } GC_oom_func GC_oom_fn=GC_default_oom_fn; #ifdef CAN_HANDLE_FORK #ifdef HANDLE_FORK GC_INNER int GC_handle_fork=1; #else GC_INNER int GC_handle_fork=FALSE; #endif #elif!defined(HAVE_NO_FORK) GC_API void GC_CALL GC_atfork_prepare(void) { #ifdef THREADS ABORT("fork()handling unsupported"); #endif } GC_API void GC_CALL GC_atfork_parent(void) { } GC_API void GC_CALL GC_atfork_child(void) { } #endif GC_API void GC_CALL GC_set_handle_fork(int value GC_ATTR_UNUSED) { #ifdef CAN_HANDLE_FORK if (!GC_is_initialized) GC_handle_fork=value>=-1?value:1; #elif defined(THREADS)||(defined(DARWIN)&&defined(MPROTECT_VDB)) if (!GC_is_initialized&&value){ #ifndef SMALL_CONFIG GC_init(); #ifndef THREADS if (GC_manual_vdb) return; #endif #endif ABORT("fork()handling unsupported"); } #else #endif } STATIC void GC_init_size_map(void) { size_t i; GC_size_map[0]=1; for (i=1;i<=GRANULES_TO_BYTES(TINY_FREELISTS-1)- EXTRA_BYTES;i++){ GC_size_map[i]=ROUNDED_UP_GRANULES(i); #ifndef _MSC_VER GC_ASSERT(GC_size_map[i] < TINY_FREELISTS); #endif } } #ifndef SMALL_CLEAR_SIZE #define SMALL_CLEAR_SIZE 256 #endif #if defined(ALWAYS_SMALL_CLEAR_STACK)||defined(STACK_NOT_SCANNED) GC_API void*GC_CALL GC_clear_stack(void*arg) { #ifndef STACK_NOT_SCANNED word volatile dummy[SMALL_CLEAR_SIZE]; BZERO(( void*)dummy,sizeof(dummy)); #endif return arg; } #else #ifdef THREADS #define BIG_CLEAR_SIZE 2048 #else STATIC word GC_stack_last_cleared=0; STATIC ptr_t GC_min_sp=NULL; STATIC ptr_t GC_high_water=NULL; STATIC word GC_bytes_allocd_at_reset=0; #define DEGRADE_RATE 50 #endif #if defined(ASM_CLEAR_CODE) void*GC_clear_stack_inner(void*,ptr_t); #else void*GC_clear_stack_inner(void*arg, #if defined(__APPLE_CC__)&&!GC_CLANG_PREREQ(6,0) volatile #endif ptr_t limit) { #define CLEAR_SIZE 213 volatile word dummy[CLEAR_SIZE]; BZERO(( void*)dummy,sizeof(dummy)); if ((word)GC_approx_sp()COOLER_THAN (word)limit){ (void)GC_clear_stack_inner(arg,limit); } #if defined(CPPCHECK) GC_noop1(dummy[0]); #else GC_noop1(COVERT_DATAFLOW(dummy)); #endif return(arg); } #endif #ifdef THREADS GC_ATTR_NO_SANITIZE_THREAD static unsigned next_random_no(void) { static unsigned random_no=0; return++random_no % 13; } #endif GC_API void*GC_CALL GC_clear_stack(void*arg) { ptr_t sp=GC_approx_sp(); #ifdef THREADS word volatile dummy[SMALL_CLEAR_SIZE]; #endif #define SLOP 400 #define GC_SLOP 4000 #define CLEAR_THRESHOLD 100000 #ifdef THREADS if (next_random_no()==0){ ptr_t limit=sp; MAKE_HOTTER(limit,BIG_CLEAR_SIZE*sizeof(word)); limit=(ptr_t)((word)limit&~0xf); return GC_clear_stack_inner(arg,limit); } BZERO((void*)dummy,SMALL_CLEAR_SIZE*sizeof(word)); #else if (GC_gc_no > GC_stack_last_cleared){ if (GC_stack_last_cleared==0) GC_high_water=(ptr_t)GC_stackbottom; GC_min_sp=GC_high_water; GC_stack_last_cleared=GC_gc_no; GC_bytes_allocd_at_reset=GC_bytes_allocd; } MAKE_COOLER(GC_high_water,WORDS_TO_BYTES(DEGRADE_RATE)+GC_SLOP); if ((word)sp HOTTER_THAN (word)GC_high_water){ GC_high_water=sp; } MAKE_HOTTER(GC_high_water,GC_SLOP); { ptr_t limit=GC_min_sp; MAKE_HOTTER(limit,SLOP); if ((word)sp COOLER_THAN (word)limit){ limit=(ptr_t)((word)limit&~0xf); GC_min_sp=sp; return GC_clear_stack_inner(arg,limit); } } if (GC_bytes_allocd - GC_bytes_allocd_at_reset > CLEAR_THRESHOLD){ GC_min_sp=sp; MAKE_HOTTER(GC_min_sp,CLEAR_THRESHOLD/4); if ((word)GC_min_sp HOTTER_THAN (word)GC_high_water) GC_min_sp=GC_high_water; GC_bytes_allocd_at_reset=GC_bytes_allocd; } #endif return arg; } #endif GC_API void*GC_CALL GC_base(void*p) { ptr_t r; struct hblk*h; bottom_index*bi; hdr*candidate_hdr; r=(ptr_t)p; if (!EXPECT(GC_is_initialized,TRUE))return 0; h=HBLKPTR(r); GET_BI(r,bi); candidate_hdr=HDR_FROM_BI(bi,r); if (candidate_hdr==0)return(0); while (IS_FORWARDING_ADDR_OR_NIL(candidate_hdr)){ h=FORWARDED_ADDR(h,candidate_hdr); r=(ptr_t)h; candidate_hdr=HDR(h); } if (HBLK_IS_FREE(candidate_hdr))return(0); r=(ptr_t)((word)r&~(WORDS_TO_BYTES(1)- 1)); { size_t offset=HBLKDISPL(r); word sz=candidate_hdr->hb_sz; size_t obj_displ=offset % sz; ptr_t limit; r-=obj_displ; limit=r+sz; if ((word)limit > (word)(h+1)&&sz<=HBLKSIZE){ return(0); } if ((word)p>=(word)limit)return(0); } return((void*)r); } GC_API int GC_CALL GC_is_heap_ptr(const void*p) { bottom_index*bi; GC_ASSERT(GC_is_initialized); GET_BI(p,bi); return HDR_FROM_BI(bi,p)!=0; } GC_API size_t GC_CALL GC_size(const void*p) { hdr*hhdr=HDR(p); return (size_t)hhdr->hb_sz; } GC_API size_t GC_CALL GC_get_heap_size(void) { return (size_t)(GC_heapsize - GC_unmapped_bytes); } GC_API size_t GC_CALL GC_get_free_bytes(void) { return (size_t)(GC_large_free_bytes - GC_unmapped_bytes); } GC_API size_t GC_CALL GC_get_unmapped_bytes(void) { return (size_t)GC_unmapped_bytes; } GC_API size_t GC_CALL GC_get_bytes_since_gc(void) { return (size_t)GC_bytes_allocd; } GC_API size_t GC_CALL GC_get_total_bytes(void) { return (size_t)(GC_bytes_allocd+GC_bytes_allocd_before_gc); } #ifndef GC_GET_HEAP_USAGE_NOT_NEEDED GC_API size_t GC_CALL GC_get_size_map_at(int i) { if ((unsigned)i > MAXOBJBYTES) return GC_SIZE_MAX; return GRANULES_TO_BYTES(GC_size_map[i]); } GC_API void GC_CALL GC_get_heap_usage_safe(GC_word*pheap_size, GC_word*pfree_bytes,GC_word*punmapped_bytes, GC_word*pbytes_since_gc,GC_word*ptotal_bytes) { DCL_LOCK_STATE; LOCK(); if (pheap_size!=NULL) *pheap_size=GC_heapsize - GC_unmapped_bytes; if (pfree_bytes!=NULL) *pfree_bytes=GC_large_free_bytes - GC_unmapped_bytes; if (punmapped_bytes!=NULL) *punmapped_bytes=GC_unmapped_bytes; if (pbytes_since_gc!=NULL) *pbytes_since_gc=GC_bytes_allocd; if (ptotal_bytes!=NULL) *ptotal_bytes=GC_bytes_allocd+GC_bytes_allocd_before_gc; UNLOCK(); } GC_INNER word GC_reclaimed_bytes_before_gc=0; static void fill_prof_stats(struct GC_prof_stats_s*pstats) { pstats->heapsize_full=GC_heapsize; pstats->free_bytes_full=GC_large_free_bytes; pstats->unmapped_bytes=GC_unmapped_bytes; pstats->bytes_allocd_since_gc=GC_bytes_allocd; pstats->allocd_bytes_before_gc=GC_bytes_allocd_before_gc; pstats->non_gc_bytes=GC_non_gc_bytes; pstats->gc_no=GC_gc_no; #ifdef PARALLEL_MARK pstats->markers_m1=(word)((signed_word)GC_markers_m1); #else pstats->markers_m1=0; #endif pstats->bytes_reclaimed_since_gc=GC_bytes_found > 0? (word)GC_bytes_found:0; pstats->reclaimed_bytes_before_gc=GC_reclaimed_bytes_before_gc; pstats->expl_freed_bytes_since_gc=GC_bytes_freed; } #include GC_API size_t GC_CALL GC_get_prof_stats(struct GC_prof_stats_s*pstats, size_t stats_sz) { struct GC_prof_stats_s stats; DCL_LOCK_STATE; LOCK(); fill_prof_stats(stats_sz>=sizeof(stats)?pstats:&stats); UNLOCK(); if (stats_sz==sizeof(stats)){ return sizeof(stats); } else if (stats_sz > sizeof(stats)){ memset((char*)pstats+sizeof(stats),0xff,stats_sz - sizeof(stats)); return sizeof(stats); } else { if (EXPECT(stats_sz > 0,TRUE)) BCOPY(&stats,pstats,stats_sz); return stats_sz; } } #ifdef THREADS GC_API size_t GC_CALL GC_get_prof_stats_unsafe( struct GC_prof_stats_s*pstats, size_t stats_sz) { struct GC_prof_stats_s stats; if (stats_sz>=sizeof(stats)){ fill_prof_stats(pstats); if (stats_sz > sizeof(stats)) memset((char*)pstats+sizeof(stats),0xff, stats_sz - sizeof(stats)); return sizeof(stats); } else { if (EXPECT(stats_sz > 0,TRUE)){ fill_prof_stats(&stats); BCOPY(&stats,pstats,stats_sz); } return stats_sz; } } #endif #endif #if defined(GC_DARWIN_THREADS)||defined(GC_OPENBSD_UTHREADS)||defined(GC_WIN32_THREADS)||(defined(NACL)&&defined(THREADS)) GC_API void GC_CALL GC_set_suspend_signal(int sig GC_ATTR_UNUSED) { } GC_API void GC_CALL GC_set_thr_restart_signal(int sig GC_ATTR_UNUSED) { } GC_API int GC_CALL GC_get_suspend_signal(void) { return -1; } GC_API int GC_CALL GC_get_thr_restart_signal(void) { return -1; } #endif #if!defined(_MAX_PATH)&&(defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32)) #define _MAX_PATH MAX_PATH #endif #ifdef GC_READ_ENV_FILE STATIC char*GC_envfile_content=NULL; STATIC unsigned GC_envfile_length=0; #ifndef GC_ENVFILE_MAXLEN #define GC_ENVFILE_MAXLEN 0x4000 #endif #define GC_ENV_FILE_EXT ".gc.env" STATIC void GC_envfile_init(void) { #if defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32) HANDLE hFile; char*content; unsigned ofs; unsigned len; DWORD nBytesRead; TCHAR path[_MAX_PATH+0x10]; len=(unsigned)GetModuleFileName(NULL,path, _MAX_PATH+1); if (len > 4&&path[len - 4]==(TCHAR)'.'){ len-=4; } BCOPY(TEXT(GC_ENV_FILE_EXT),&path[len],sizeof(TEXT(GC_ENV_FILE_EXT))); hFile=CreateFile(path,GENERIC_READ, FILE_SHARE_READ|FILE_SHARE_WRITE, NULL,OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,NULL); if (hFile==INVALID_HANDLE_VALUE) return; len=(unsigned)GetFileSize(hFile,NULL); if (len<=1||len>=GC_ENVFILE_MAXLEN){ CloseHandle(hFile); return; } GC_ASSERT(GC_page_size!=0); content=(char*)GET_MEM(ROUNDUP_PAGESIZE_IF_MMAP((size_t)len+1)); if (content==NULL){ CloseHandle(hFile); return; } ofs=0; nBytesRead=(DWORD)-1L; while (ReadFile(hFile,content+ofs,len - ofs+1,&nBytesRead, NULL)&&nBytesRead!=0){ if ((ofs+=nBytesRead)> len) break; } CloseHandle(hFile); if (ofs!=len||nBytesRead!=0) return; content[ofs]='\0'; while (ofs--> 0){ if (content[ofs]=='\r'||content[ofs]=='\n') content[ofs]='\0'; } GC_ASSERT(NULL==GC_envfile_content); GC_envfile_length=len+1; GC_envfile_content=content; #endif } GC_INNER char*GC_envfile_getenv(const char*name) { char*p; char*end_of_content; unsigned namelen; #ifndef NO_GETENV p=getenv(name); if (p!=NULL) return*p!='\0'?p:NULL; #endif p=GC_envfile_content; if (p==NULL) return NULL; namelen=strlen(name); if (namelen==0) return NULL; for (end_of_content=p+GC_envfile_length; p!=end_of_content;p+=strlen(p)+1){ if (strncmp(p,name,namelen)==0&&*(p+=namelen)=='='){ p++; return*p!='\0'?p:NULL; } } return NULL; } #endif GC_INNER GC_bool GC_is_initialized=FALSE; GC_API int GC_CALL GC_is_init_called(void) { return GC_is_initialized; } #if defined(GC_WIN32_THREADS)&&((defined(MSWIN32)&&!defined(CONSOLE_LOG))||defined(MSWINCE)) GC_INNER CRITICAL_SECTION GC_write_cs; #endif #ifndef DONT_USE_ATEXIT #if!defined(PCR)&&!defined(SMALL_CONFIG) static GC_bool skip_gc_atexit=FALSE; #else #define skip_gc_atexit FALSE #endif STATIC void GC_exit_check(void) { if (GC_find_leak&&!skip_gc_atexit){ #ifdef THREADS GC_in_thread_creation=TRUE; GC_gcollect(); GC_in_thread_creation=FALSE; #else GC_gcollect(); #endif } } #endif #if defined(UNIX_LIKE)&&!defined(NO_DEBUGGING) static void looping_handler(int sig) { GC_err_printf("Caught signal %d:looping in handler\n",sig); for (;;){ } } static GC_bool installed_looping_handler=FALSE; static void maybe_install_looping_handler(void) { if (!installed_looping_handler&&0!=GETENV("GC_LOOP_ON_ABORT")){ GC_set_and_save_fault_handler(looping_handler); installed_looping_handler=TRUE; } } #else #define maybe_install_looping_handler() #endif #define GC_DEFAULT_STDOUT_FD 1 #define GC_DEFAULT_STDERR_FD 2 #if!defined(OS2)&&!defined(MACOS)&&!defined(GC_ANDROID_LOG)&&!defined(NN_PLATFORM_CTR)&&!defined(NINTENDO_SWITCH)&&(!defined(MSWIN32)||defined(CONSOLE_LOG))&&!defined(MSWINCE) STATIC int GC_stdout=GC_DEFAULT_STDOUT_FD; STATIC int GC_stderr=GC_DEFAULT_STDERR_FD; STATIC int GC_log=GC_DEFAULT_STDERR_FD; #ifndef MSWIN32 GC_API void GC_CALL GC_set_log_fd(int fd) { GC_log=fd; } #endif #endif #ifdef MSGBOX_ON_ERROR STATIC void GC_win32_MessageBoxA(const char*msg,const char*caption, unsigned flags) { #ifndef DONT_USE_USER32_DLL (void)MessageBoxA(NULL,msg,caption,flags); #else HINSTANCE hU32=LoadLibrary(TEXT("user32.dll")); if (hU32){ FARPROC pfn=GetProcAddress(hU32,"MessageBoxA"); if (pfn) (void)(*(int (WINAPI*)(HWND,LPCSTR,LPCSTR,UINT))(word)pfn)( NULL,msg,caption,flags); (void)FreeLibrary(hU32); } #endif } #endif #if defined(THREADS)&&defined(UNIX_LIKE)&&!defined(NO_GETCONTEXT) static void callee_saves_pushed_dummy_fn(ptr_t data GC_ATTR_UNUSED, void*context GC_ATTR_UNUSED){} #endif #ifndef SMALL_CONFIG #ifdef MANUAL_VDB static GC_bool manual_vdb_allowed=TRUE; #else static GC_bool manual_vdb_allowed=FALSE; #endif GC_API void GC_CALL GC_set_manual_vdb_allowed(int value) { manual_vdb_allowed=(GC_bool)value; } GC_API int GC_CALL GC_get_manual_vdb_allowed(void) { return (int)manual_vdb_allowed; } #endif STATIC word GC_parse_mem_size_arg(const char*str) { word result=0; if (*str!='\0'){ char*endptr; char ch; result=(word)STRTOULL(str,&endptr,10); ch=*endptr; if (ch!='\0'){ if (*(endptr+1)!='\0') return 0; switch (ch){ case 'K': case 'k': result<<=10; break; case 'M': case 'm': result<<=20; break; case 'G': case 'g': result<<=30; break; default: result=0; } } } return result; } #define GC_LOG_STD_NAME "gc.log" GC_API void GC_CALL GC_init(void) { word initial_heap_sz; IF_CANCEL(int cancel_state;) #if defined(GC_ASSERTIONS)&&defined(GC_ALWAYS_MULTITHREADED) DCL_LOCK_STATE; #endif if (EXPECT(GC_is_initialized,TRUE))return; #ifdef REDIRECT_MALLOC { static GC_bool init_started=FALSE; if (init_started) ABORT("Redirected malloc()called during GC init"); init_started=TRUE; } #endif #if defined(GC_INITIAL_HEAP_SIZE)&&!defined(CPPCHECK) initial_heap_sz=GC_INITIAL_HEAP_SIZE; #else initial_heap_sz=MINHINCR*HBLKSIZE; #endif DISABLE_CANCEL(cancel_state); #ifdef THREADS #ifndef GC_ALWAYS_MULTITHREADED GC_ASSERT(!GC_need_to_lock); #endif #ifdef SN_TARGET_PS3 { pthread_mutexattr_t mattr; if (0!=pthread_mutexattr_init(&mattr)){ ABORT("pthread_mutexattr_init failed"); } if (0!=pthread_mutex_init(&GC_allocate_ml,&mattr)){ ABORT("pthread_mutex_init failed"); } (void)pthread_mutexattr_destroy(&mattr); } #endif #endif #if defined(GC_WIN32_THREADS)&&!defined(GC_PTHREADS) #ifndef SPIN_COUNT #define SPIN_COUNT 4000 #endif #ifdef MSWINRT_FLAVOR InitializeCriticalSectionAndSpinCount(&GC_allocate_ml,SPIN_COUNT); #else { #ifndef MSWINCE FARPROC pfn=0; HMODULE hK32=GetModuleHandle(TEXT("kernel32.dll")); if (hK32) pfn=GetProcAddress(hK32, "InitializeCriticalSectionAndSpinCount"); if (pfn){ (*(BOOL (WINAPI*)(LPCRITICAL_SECTION,DWORD))(word)pfn)( &GC_allocate_ml,SPIN_COUNT); } else #endif InitializeCriticalSection(&GC_allocate_ml); } #endif #endif #if defined(GC_WIN32_THREADS)&&((defined(MSWIN32)&&!defined(CONSOLE_LOG))||defined(MSWINCE)) InitializeCriticalSection(&GC_write_cs); #endif GC_setpagesize(); #ifdef MSWIN32 GC_init_win32(); #endif #ifdef GC_READ_ENV_FILE GC_envfile_init(); #endif #if!defined(NO_CLOCK)||!defined(SMALL_CONFIG) #ifdef GC_PRINT_VERBOSE_STATS GC_print_stats=VERBOSE; #else if (0!=GETENV("GC_PRINT_VERBOSE_STATS")){ GC_print_stats=VERBOSE; } else if (0!=GETENV("GC_PRINT_STATS")){ GC_print_stats=1; } #endif #endif #if ((defined(UNIX_LIKE)&&!defined(GC_ANDROID_LOG))||(defined(CONSOLE_LOG)&&defined(MSWIN32))||defined(CYGWIN32)||defined(SYMBIAN))&&!defined(SMALL_CONFIG) { char*file_name=TRUSTED_STRING(GETENV("GC_LOG_FILE")); #ifdef GC_LOG_TO_FILE_ALWAYS if (NULL==file_name) file_name=GC_LOG_STD_NAME; #else if (0!=file_name) #endif { #if defined(_MSC_VER) int log_d=_open(file_name,O_CREAT|O_WRONLY|O_APPEND); #else int log_d=open(file_name,O_CREAT|O_WRONLY|O_APPEND,0644); #endif if (log_d < 0){ GC_err_printf("Failed to open %s as log file\n",file_name); } else { char*str; GC_log=log_d; str=GETENV("GC_ONLY_LOG_TO_FILE"); #ifdef GC_ONLY_LOG_TO_FILE if (str!=NULL&&*str=='0'&&*(str+1)=='\0') #else if (str==NULL||(*str=='0'&&*(str+1)=='\0')) #endif { GC_stdout=log_d; GC_stderr=log_d; } } } } #endif #if!defined(NO_DEBUGGING)&&!defined(GC_DUMP_REGULARLY) if (0!=GETENV("GC_DUMP_REGULARLY")){ GC_dump_regularly=TRUE; } #endif #ifdef KEEP_BACK_PTRS { char*backtraces_string=GETENV("GC_BACKTRACES"); if (0!=backtraces_string){ GC_backtraces=atol(backtraces_string); if (backtraces_string[0]=='\0')GC_backtraces=1; } } #endif if (0!=GETENV("GC_FIND_LEAK")){ GC_find_leak=1; } #ifndef SHORT_DBG_HDRS if (0!=GETENV("GC_FINDLEAK_DELAY_FREE")){ GC_findleak_delay_free=TRUE; } #endif if (0!=GETENV("GC_ALL_INTERIOR_POINTERS")){ GC_all_interior_pointers=1; } if (0!=GETENV("GC_DONT_GC")){ GC_dont_gc=1; } if (0!=GETENV("GC_PRINT_BACK_HEIGHT")){ GC_print_back_height=TRUE; } if (0!=GETENV("GC_NO_BLACKLIST_WARNING")){ GC_large_alloc_warn_interval=LONG_MAX; } { char*addr_string=GETENV("GC_TRACE"); if (0!=addr_string){ #ifndef ENABLE_TRACE WARN("Tracing not enabled:Ignoring GC_TRACE value\n",0); #else word addr=(word)STRTOULL(addr_string,NULL,16); if (addr < 0x1000) WARN("Unlikely trace address:%p\n",(void*)addr); GC_trace_addr=(ptr_t)addr; #endif } } #ifdef GC_COLLECT_AT_MALLOC { char*string=GETENV("GC_COLLECT_AT_MALLOC"); if (0!=string){ size_t min_lb=(size_t)STRTOULL(string,NULL,10); if (min_lb > 0) GC_dbg_collect_at_malloc_min_lb=min_lb; } } #endif #if!defined(GC_DISABLE_INCREMENTAL)&&!defined(NO_CLOCK) { char*time_limit_string=GETENV("GC_PAUSE_TIME_TARGET"); if (0!=time_limit_string){ long time_limit=atol(time_limit_string); if (time_limit > 0){ GC_time_limit=time_limit; } } } #endif #ifndef SMALL_CONFIG { char*full_freq_string=GETENV("GC_FULL_FREQUENCY"); if (full_freq_string!=NULL){ int full_freq=atoi(full_freq_string); if (full_freq > 0) GC_full_freq=full_freq; } } #endif { char*interval_string=GETENV("GC_LARGE_ALLOC_WARN_INTERVAL"); if (0!=interval_string){ long interval=atol(interval_string); if (interval<=0){ WARN("GC_LARGE_ALLOC_WARN_INTERVAL environment variable has " "bad value:Ignoring\n",0); } else { GC_large_alloc_warn_interval=interval; } } } { char*space_divisor_string=GETENV("GC_FREE_SPACE_DIVISOR"); if (space_divisor_string!=NULL){ int space_divisor=atoi(space_divisor_string); if (space_divisor > 0) GC_free_space_divisor=(unsigned)space_divisor; } } #ifdef USE_MUNMAP { char*string=GETENV("GC_UNMAP_THRESHOLD"); if (string!=NULL){ if (*string=='0'&&*(string+1)=='\0'){ GC_unmap_threshold=0; } else { int unmap_threshold=atoi(string); if (unmap_threshold > 0) GC_unmap_threshold=unmap_threshold; } } } { char*string=GETENV("GC_FORCE_UNMAP_ON_GCOLLECT"); if (string!=NULL){ if (*string=='0'&&*(string+1)=='\0'){ GC_force_unmap_on_gcollect=FALSE; } else { GC_force_unmap_on_gcollect=TRUE; } } } { char*string=GETENV("GC_USE_ENTIRE_HEAP"); if (string!=NULL){ if (*string=='0'&&*(string+1)=='\0'){ GC_use_entire_heap=FALSE; } else { GC_use_entire_heap=TRUE; } } } #endif #if!defined(NO_DEBUGGING)&&!defined(NO_CLOCK) GET_TIME(GC_init_time); #endif maybe_install_looping_handler(); #if ALIGNMENT > GC_DS_TAGS if (EXTRA_BYTES!=0) GC_obj_kinds[NORMAL].ok_descriptor=(word)(-ALIGNMENT)|GC_DS_LENGTH; #endif GC_exclude_static_roots_inner(beginGC_arrays,endGC_arrays); GC_exclude_static_roots_inner(beginGC_obj_kinds,endGC_obj_kinds); #ifdef SEPARATE_GLOBALS GC_exclude_static_roots_inner(beginGC_objfreelist,endGC_objfreelist); GC_exclude_static_roots_inner(beginGC_aobjfreelist,endGC_aobjfreelist); #endif #if defined(USE_PROC_FOR_LIBRARIES)&&defined(GC_LINUX_THREADS) WARN("USE_PROC_FOR_LIBRARIES+GC_LINUX_THREADS performs poorly.\n",0); #endif #if defined(SEARCH_FOR_DATA_START) GC_init_linux_data_start(); #endif #if defined(NETBSD)&&defined(__ELF__) GC_init_netbsd_elf(); #endif #if!defined(THREADS)||defined(GC_PTHREADS)||defined(NN_PLATFORM_CTR)||defined(NINTENDO_SWITCH)||defined(GC_WIN32_THREADS)||defined(GC_SOLARIS_THREADS) if (GC_stackbottom==0){ GC_stackbottom=GC_get_main_stack_base(); #if (defined(LINUX)||defined(HPUX))&&defined(IA64) GC_register_stackbottom=GC_get_register_stack_base(); #endif } else { #if (defined(LINUX)||defined(HPUX))&&defined(IA64) if (GC_register_stackbottom==0){ WARN("GC_register_stackbottom should be set with GC_stackbottom\n",0); GC_register_stackbottom=GC_get_register_stack_base(); } #endif } #endif #if!defined(CPPCHECK) GC_STATIC_ASSERT(sizeof(ptr_t)==sizeof(word)); GC_STATIC_ASSERT(sizeof(signed_word)==sizeof(word)); #if!defined(_AUX_SOURCE)||defined(__GNUC__) GC_STATIC_ASSERT((word)(-1)> (word)0); #endif GC_STATIC_ASSERT((signed_word)(-1)< (signed_word)0); #endif GC_STATIC_ASSERT(sizeof (struct hblk)==HBLKSIZE); #ifndef THREADS GC_ASSERT(!((word)GC_stackbottom HOTTER_THAN (word)GC_approx_sp())); #endif #ifndef GC_DISABLE_INCREMENTAL if (GC_incremental||0!=GETENV("GC_ENABLE_INCREMENTAL")){ #if defined(BASE_ATOMIC_OPS_EMULATED)||defined(CHECKSUMS)||defined(REDIRECT_MALLOC)||defined(REDIRECT_MALLOC_IN_HEADER)||defined(SMALL_CONFIG) #else if (manual_vdb_allowed){ GC_manual_vdb=TRUE; GC_incremental=TRUE; } else #endif { GC_incremental=GC_dirty_init(); GC_ASSERT(GC_bytes_allocd==0); } } #endif if (GC_REGISTER_MAIN_STATIC_DATA())GC_register_data_segments(); GC_init_headers(); GC_bl_init(); GC_mark_init(); { char*sz_str=GETENV("GC_INITIAL_HEAP_SIZE"); if (sz_str!=NULL){ initial_heap_sz=GC_parse_mem_size_arg(sz_str); if (initial_heap_sz<=MINHINCR*HBLKSIZE){ WARN("Bad initial heap size %s - ignoring it.\n",sz_str); } } } { char*sz_str=GETENV("GC_MAXIMUM_HEAP_SIZE"); if (sz_str!=NULL){ word max_heap_sz=GC_parse_mem_size_arg(sz_str); if (max_heap_sz < initial_heap_sz){ WARN("Bad maximum heap size %s - ignoring it.\n",sz_str); } if (0==GC_max_retries)GC_max_retries=2; GC_set_max_heap_size(max_heap_sz); } } #if defined(GC_ASSERTIONS)&&defined(GC_ALWAYS_MULTITHREADED) LOCK(); #endif if (!GC_expand_hp_inner(divHBLKSZ(initial_heap_sz))){ GC_err_printf("Can't start up:not enough memory\n"); EXIT(); } else { GC_requested_heapsize+=initial_heap_sz; } if (GC_all_interior_pointers) GC_initialize_offsets(); GC_register_displacement_inner(0L); #if defined(GC_LINUX_THREADS)&&defined(REDIRECT_MALLOC) if (!GC_all_interior_pointers){ GC_register_displacement_inner(sizeof(void*)); } #endif GC_init_size_map(); #ifdef PCR if (PCR_IL_Lock(PCR_Bool_false,PCR_allSigsBlocked,PCR_waitForever) !=PCR_ERes_okay){ ABORT("Can't lock load state"); } else if (PCR_IL_Unlock()!=PCR_ERes_okay){ ABORT("Can't unlock load state"); } PCR_IL_Unlock(); GC_pcr_install(); #endif GC_is_initialized=TRUE; #if defined(GC_PTHREADS)||defined(GC_WIN32_THREADS) GC_thr_init(); #ifdef PARALLEL_MARK #if defined(GC_ASSERTIONS)&&defined(GC_ALWAYS_MULTITHREADED) UNLOCK(); #endif GC_start_mark_threads_inner(); #if defined(GC_ASSERTIONS)&&defined(GC_ALWAYS_MULTITHREADED) LOCK(); #endif #endif #endif COND_DUMP; if (!GC_dont_precollect||GC_incremental){ GC_gcollect_inner(); } #if defined(GC_ASSERTIONS)&&defined(GC_ALWAYS_MULTITHREADED) UNLOCK(); #endif #if defined(THREADS)&&defined(UNIX_LIKE)&&!defined(NO_GETCONTEXT) if (GC_dont_gc||GC_dont_precollect) GC_with_callee_saves_pushed(callee_saves_pushed_dummy_fn,NULL); #endif #ifndef DONT_USE_ATEXIT if (GC_find_leak){ atexit(GC_exit_check); } #endif #if defined(PARALLEL_MARK)||defined(THREAD_LOCAL_ALLOC)||(defined(GC_ALWAYS_MULTITHREADED)&&defined(GC_WIN32_THREADS)&&!defined(GC_NO_THREADS_DISCOVERY)) GC_init_parallel(); #endif #if defined(DYNAMIC_LOADING)&&defined(DARWIN) GC_init_dyld(); #endif RESTORE_CANCEL(cancel_state); } GC_API void GC_CALL GC_enable_incremental(void) { #if!defined(GC_DISABLE_INCREMENTAL)&&!defined(KEEP_BACK_PTRS) DCL_LOCK_STATE; if (!GC_find_leak&&0==GETENV("GC_DISABLE_INCREMENTAL")){ LOCK(); if (!GC_incremental){ GC_setpagesize(); maybe_install_looping_handler(); if (!GC_is_initialized){ UNLOCK(); GC_incremental=TRUE; GC_init(); LOCK(); } else { #if!defined(BASE_ATOMIC_OPS_EMULATED)&&!defined(CHECKSUMS)&&!defined(REDIRECT_MALLOC)&&!defined(REDIRECT_MALLOC_IN_HEADER)&&!defined(SMALL_CONFIG) if (manual_vdb_allowed){ GC_manual_vdb=TRUE; GC_incremental=TRUE; } else #endif { GC_incremental=GC_dirty_init(); } } if (GC_incremental&&!GC_dont_gc){ IF_CANCEL(int cancel_state;) DISABLE_CANCEL(cancel_state); if (GC_bytes_allocd > 0){ GC_gcollect_inner(); } GC_read_dirty(FALSE); RESTORE_CANCEL(cancel_state); } } UNLOCK(); return; } #endif GC_init(); } #if defined(THREADS) GC_API void GC_CALL GC_start_mark_threads(void) { #if defined(PARALLEL_MARK)&&defined(CAN_HANDLE_FORK)&&!defined(THREAD_SANITIZER) IF_CANCEL(int cancel_state;) DISABLE_CANCEL(cancel_state); GC_start_mark_threads_inner(); RESTORE_CANCEL(cancel_state); #else GC_ASSERT(I_DONT_HOLD_LOCK()); #endif } #endif GC_API void GC_CALL GC_deinit(void) { if (GC_is_initialized){ GC_is_initialized=FALSE; #if defined(GC_WIN32_THREADS)&&(defined(MSWIN32)||defined(MSWINCE)) #if!defined(CONSOLE_LOG)||defined(MSWINCE) DeleteCriticalSection(&GC_write_cs); #endif DeleteCriticalSection(&GC_allocate_ml); #endif } } #if (defined(MSWIN32)&&!defined(CONSOLE_LOG))||defined(MSWINCE) #if defined(_MSC_VER)&&defined(_DEBUG)&&!defined(MSWINCE) #include #endif STATIC HANDLE GC_log=0; #ifdef THREADS #if defined(PARALLEL_MARK)&&!defined(GC_ALWAYS_MULTITHREADED) #define IF_NEED_TO_LOCK(x)if (GC_parallel||GC_need_to_lock)x #else #define IF_NEED_TO_LOCK(x)if (GC_need_to_lock)x #endif #else #define IF_NEED_TO_LOCK(x) #endif #ifdef MSWINRT_FLAVOR #include DECLSPEC_IMPORT HRESULT WINAPI RoGetActivationFactory( HSTRING activatableClassId, REFIID iid,void**factory); static GC_bool getWinRTLogPath(wchar_t*buf,size_t bufLen) { static const GUID kIID_IApplicationDataStatics={ 0x5612147B,0xE843,0x45E3, 0x94,0xD8,0x06,0x16,0x9E,0x3C,0x8E,0x17 }; static const GUID kIID_IStorageItem={ 0x4207A996,0xCA2F,0x42F7, 0xBD,0xE8,0x8B,0x10,0x45,0x7A,0x7F,0x30 }; GC_bool result=FALSE; HSTRING_HEADER appDataClassNameHeader; HSTRING appDataClassName; __x_ABI_CWindows_CStorage_CIApplicationDataStatics*appDataStatics=0; GC_ASSERT(bufLen > 0); if (SUCCEEDED(WindowsCreateStringReference( RuntimeClass_Windows_Storage_ApplicationData, (sizeof(RuntimeClass_Windows_Storage_ApplicationData)-1) /sizeof(wchar_t), &appDataClassNameHeader,&appDataClassName)) &&SUCCEEDED(RoGetActivationFactory(appDataClassName, &kIID_IApplicationDataStatics, &appDataStatics))){ __x_ABI_CWindows_CStorage_CIApplicationData*appData=NULL; __x_ABI_CWindows_CStorage_CIStorageFolder*tempFolder=NULL; __x_ABI_CWindows_CStorage_CIStorageItem*tempFolderItem=NULL; HSTRING tempPath=NULL; if (SUCCEEDED(appDataStatics->lpVtbl->get_Current(appDataStatics, &appData)) &&SUCCEEDED(appData->lpVtbl->get_TemporaryFolder(appData, &tempFolder)) &&SUCCEEDED(tempFolder->lpVtbl->QueryInterface(tempFolder, &kIID_IStorageItem, &tempFolderItem)) &&SUCCEEDED(tempFolderItem->lpVtbl->get_Path(tempFolderItem, &tempPath))){ UINT32 tempPathLen; const wchar_t*tempPathBuf= WindowsGetStringRawBuffer(tempPath,&tempPathLen); buf[0]='\0'; if (wcsncat_s(buf,bufLen,tempPathBuf,tempPathLen)==0 &&wcscat_s(buf,bufLen,L"\\")==0 &&wcscat_s(buf,bufLen,TEXT(GC_LOG_STD_NAME))==0) result=TRUE; WindowsDeleteString(tempPath); } if (tempFolderItem!=NULL) tempFolderItem->lpVtbl->Release(tempFolderItem); if (tempFolder!=NULL) tempFolder->lpVtbl->Release(tempFolder); if (appData!=NULL) appData->lpVtbl->Release(appData); appDataStatics->lpVtbl->Release(appDataStatics); } return result; } #endif STATIC HANDLE GC_CreateLogFile(void) { HANDLE hFile; #ifdef MSWINRT_FLAVOR TCHAR pathBuf[_MAX_PATH+0x10]; hFile=INVALID_HANDLE_VALUE; if (getWinRTLogPath(pathBuf,_MAX_PATH+1)){ CREATEFILE2_EXTENDED_PARAMETERS extParams; BZERO(&extParams,sizeof(extParams)); extParams.dwSize=sizeof(extParams); extParams.dwFileAttributes=FILE_ATTRIBUTE_NORMAL; extParams.dwFileFlags=GC_print_stats==VERBOSE?0 :FILE_FLAG_WRITE_THROUGH; hFile=CreateFile2(pathBuf,GENERIC_WRITE,FILE_SHARE_READ, CREATE_ALWAYS,&extParams); } #else TCHAR*logPath; #if defined(NO_GETENV_WIN32)&&defined(CPPCHECK) #define appendToFile FALSE #else BOOL appendToFile=FALSE; #endif #if!defined(NO_GETENV_WIN32)||!defined(OLD_WIN32_LOG_FILE) TCHAR pathBuf[_MAX_PATH+0x10]; logPath=pathBuf; #endif #ifndef NO_GETENV_WIN32 if (GetEnvironmentVariable(TEXT("GC_LOG_FILE"),pathBuf, _MAX_PATH+1)- 1U < (DWORD)_MAX_PATH){ appendToFile=TRUE; } else #endif { #ifdef OLD_WIN32_LOG_FILE logPath=TEXT(GC_LOG_STD_NAME); #else int len=(int)GetModuleFileName(NULL,pathBuf, _MAX_PATH+1); if (len > 4&&pathBuf[len - 4]==(TCHAR)'.'){ len-=4; } BCOPY(TEXT(".")TEXT(GC_LOG_STD_NAME),&pathBuf[len], sizeof(TEXT(".")TEXT(GC_LOG_STD_NAME))); #endif } hFile=CreateFile(logPath,GENERIC_WRITE,FILE_SHARE_READ, NULL, appendToFile?OPEN_ALWAYS:CREATE_ALWAYS, GC_print_stats==VERBOSE?FILE_ATTRIBUTE_NORMAL: FILE_ATTRIBUTE_NORMAL|FILE_FLAG_WRITE_THROUGH, NULL); #ifndef NO_GETENV_WIN32 if (appendToFile&&hFile!=INVALID_HANDLE_VALUE){ LONG posHigh=0; (void)SetFilePointer(hFile,0,&posHigh,FILE_END); } #endif #undef appendToFile #endif return hFile; } STATIC int GC_write(const char*buf,size_t len) { BOOL res; DWORD written; #if defined(THREADS)&&defined(GC_ASSERTIONS) static GC_bool inside_write=FALSE; if (inside_write) return -1; #endif if (len==0) return 0; IF_NEED_TO_LOCK(EnterCriticalSection(&GC_write_cs)); #if defined(THREADS)&&defined(GC_ASSERTIONS) if (GC_write_disabled){ inside_write=TRUE; ABORT("Assertion failure:GC_write called with write_disabled"); } #endif if (GC_log==0){ GC_log=GC_CreateLogFile(); } if (GC_log==INVALID_HANDLE_VALUE){ IF_NEED_TO_LOCK(LeaveCriticalSection(&GC_write_cs)); #ifdef NO_DEBUGGING return 0; #else return -1; #endif } res=WriteFile(GC_log,buf,(DWORD)len,&written,NULL); #if defined(_MSC_VER)&&defined(_DEBUG)&&!defined(NO_CRT) #ifdef MSWINCE { WCHAR wbuf[1024]; wbuf[MultiByteToWideChar(CP_ACP,0, buf,len,wbuf, sizeof(wbuf)/sizeof(wbuf[0])- 1)]=0; OutputDebugStringW(wbuf); } #else _CrtDbgReport(_CRT_WARN,NULL,0,NULL,"%.*s",len,buf); #endif #endif IF_NEED_TO_LOCK(LeaveCriticalSection(&GC_write_cs)); return res?(int)written:-1; } #define WRITE(f,buf,len)GC_write(buf,len) #elif defined(OS2)||defined(MACOS) STATIC FILE*GC_stdout=NULL; STATIC FILE*GC_stderr=NULL; STATIC FILE*GC_log=NULL; STATIC void GC_set_files(void) { if (GC_stdout==NULL){ GC_stdout=stdout; } if (GC_stderr==NULL){ GC_stderr=stderr; } if (GC_log==NULL){ GC_log=stderr; } } GC_INLINE int GC_write(FILE*f,const char*buf,size_t len) { int res=fwrite(buf,1,len,f); fflush(f); return res; } #define WRITE(f,buf,len)(GC_set_files(),GC_write(f,buf,len)) #elif defined(GC_ANDROID_LOG) #include #ifndef GC_ANDROID_LOG_TAG #define GC_ANDROID_LOG_TAG "BDWGC" #endif #define GC_stdout ANDROID_LOG_DEBUG #define GC_stderr ANDROID_LOG_ERROR #define GC_log GC_stdout #define WRITE(level,buf,unused_len)__android_log_write(level,GC_ANDROID_LOG_TAG,buf) #elif defined(NN_PLATFORM_CTR) int n3ds_log_write(const char*text,int length); #define WRITE(level,buf,len)n3ds_log_write(buf,len) #elif defined(NINTENDO_SWITCH) int switch_log_write(const char*text,int length); #define WRITE(level,buf,len)switch_log_write(buf,len) #else #if!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PSP2) #if!defined(AMIGA)&&!defined(MSWIN32)&&!defined(MSWIN_XBOX1)&&!defined(__CC_ARM) #include #endif #if!defined(ECOS)&&!defined(NOSYS) #include #endif #endif STATIC int GC_write(int fd,const char*buf,size_t len) { #if defined(ECOS)||defined(SN_TARGET_ORBIS)||defined(SN_TARGET_PSP2)||defined(NOSYS) #ifdef ECOS #else #endif return len; #else int bytes_written=0; IF_CANCEL(int cancel_state;) DISABLE_CANCEL(cancel_state); while ((unsigned)bytes_written < len){ #ifdef GC_SOLARIS_THREADS int result=syscall(SYS_write,fd,buf+bytes_written, len - bytes_written); #elif defined(_MSC_VER) int result=_write(fd,buf+bytes_written, (unsigned)(len - bytes_written)); #else int result=write(fd,buf+bytes_written,len - bytes_written); #endif if (-1==result){ if (EAGAIN==errno) continue; RESTORE_CANCEL(cancel_state); return(result); } bytes_written+=result; } RESTORE_CANCEL(cancel_state); return(bytes_written); #endif } #define WRITE(f,buf,len)GC_write(f,buf,len) #endif #define BUFSZ 1024 #if defined(DJGPP)||defined(__STRICT_ANSI__) #define GC_VSNPRINTF(buf,bufsz,format,args)vsprintf(buf,format,args) #elif defined(_MSC_VER) #ifdef MSWINCE #define GC_VSNPRINTF StringCchVPrintfA #else #define GC_VSNPRINTF _vsnprintf #endif #else #define GC_VSNPRINTF vsnprintf #endif #define GC_PRINTF_FILLBUF(buf,format)do { va_list args;va_start(args,format);(buf)[sizeof(buf)- 1]=0x15;(void)GC_VSNPRINTF(buf,sizeof(buf)- 1,format,args);va_end(args);if ((buf)[sizeof(buf)- 1]!=0x15)ABORT("GC_printf clobbered stack");} while (0) void GC_printf(const char*format,...) { if (!GC_quiet){ char buf[BUFSZ+1]; GC_PRINTF_FILLBUF(buf,format); #ifdef NACL (void)WRITE(GC_stdout,buf,strlen(buf)); #else if (WRITE(GC_stdout,buf,strlen(buf))< 0 #if defined(CYGWIN32)||(defined(CONSOLE_LOG)&&defined(MSWIN32)) &&GC_stdout!=GC_DEFAULT_STDOUT_FD #endif ){ ABORT("write to stdout failed"); } #endif } } void GC_err_printf(const char*format,...) { char buf[BUFSZ+1]; GC_PRINTF_FILLBUF(buf,format); GC_err_puts(buf); } void GC_log_printf(const char*format,...) { char buf[BUFSZ+1]; GC_PRINTF_FILLBUF(buf,format); #ifdef NACL (void)WRITE(GC_log,buf,strlen(buf)); #else if (WRITE(GC_log,buf,strlen(buf))< 0 #if defined(CYGWIN32)||(defined(CONSOLE_LOG)&&defined(MSWIN32)) &&GC_log!=GC_DEFAULT_STDERR_FD #endif ){ ABORT("write to GC log failed"); } #endif } #ifndef GC_ANDROID_LOG #define GC_warn_printf GC_err_printf #else GC_INNER void GC_info_log_printf(const char*format,...) { char buf[BUFSZ+1]; GC_PRINTF_FILLBUF(buf,format); (void)WRITE(ANDROID_LOG_INFO,buf,0); } GC_INNER void GC_verbose_log_printf(const char*format,...) { char buf[BUFSZ+1]; GC_PRINTF_FILLBUF(buf,format); (void)WRITE(ANDROID_LOG_VERBOSE,buf,0); } STATIC void GC_warn_printf(const char*format,...) { char buf[BUFSZ+1]; GC_PRINTF_FILLBUF(buf,format); (void)WRITE(ANDROID_LOG_WARN,buf,0); } #endif void GC_err_puts(const char*s) { (void)WRITE(GC_stderr,s,strlen(s)); } STATIC void GC_CALLBACK GC_default_warn_proc(char*msg,GC_word arg) { GC_warn_printf(msg,arg); } GC_INNER GC_warn_proc GC_current_warn_proc=GC_default_warn_proc; GC_API void GC_CALLBACK GC_ignore_warn_proc(char*msg,GC_word arg) { if (GC_print_stats){ GC_default_warn_proc(msg,arg); } } GC_API void GC_CALL GC_set_warn_proc(GC_warn_proc p) { DCL_LOCK_STATE; GC_ASSERT(NONNULL_ARG_NOT_NULL(p)); #ifdef GC_WIN32_THREADS #ifdef CYGWIN32 GC_ASSERT(GC_is_initialized); #else if (!GC_is_initialized)GC_init(); #endif #endif LOCK(); GC_current_warn_proc=p; UNLOCK(); } GC_API GC_warn_proc GC_CALL GC_get_warn_proc(void) { GC_warn_proc result; DCL_LOCK_STATE; LOCK(); result=GC_current_warn_proc; UNLOCK(); return(result); } #if!defined(PCR)&&!defined(SMALL_CONFIG) STATIC void GC_CALLBACK GC_default_on_abort(const char*msg) { #ifndef DONT_USE_ATEXIT skip_gc_atexit=TRUE; #endif if (msg!=NULL){ #ifdef MSGBOX_ON_ERROR GC_win32_MessageBoxA(msg,"Fatal error in GC",MB_ICONERROR|MB_OK); #endif #ifndef GC_ANDROID_LOG #if defined(GC_WIN32_THREADS)&&defined(GC_ASSERTIONS)&&((defined(MSWIN32)&&!defined(CONSOLE_LOG))||defined(MSWINCE)) if (!GC_write_disabled) #endif { if (WRITE(GC_stderr,msg,strlen(msg))>=0) (void)WRITE(GC_stderr,"\n",1); } #else __android_log_assert("*",GC_ANDROID_LOG_TAG,"%s\n",msg); #endif } #if!defined(NO_DEBUGGING)&&!defined(GC_ANDROID_LOG) if (GETENV("GC_LOOP_ON_ABORT")!=NULL){ for(;;){ } } #endif } GC_abort_func GC_on_abort=GC_default_on_abort; GC_API void GC_CALL GC_set_abort_func(GC_abort_func fn) { DCL_LOCK_STATE; GC_ASSERT(NONNULL_ARG_NOT_NULL(fn)); LOCK(); GC_on_abort=fn; UNLOCK(); } GC_API GC_abort_func GC_CALL GC_get_abort_func(void) { GC_abort_func fn; DCL_LOCK_STATE; LOCK(); fn=GC_on_abort; UNLOCK(); return fn; } #endif GC_API void GC_CALL GC_enable(void) { DCL_LOCK_STATE; LOCK(); GC_ASSERT(GC_dont_gc!=0); GC_dont_gc--; UNLOCK(); } GC_API void GC_CALL GC_disable(void) { DCL_LOCK_STATE; LOCK(); GC_dont_gc++; UNLOCK(); } GC_API int GC_CALL GC_is_disabled(void) { return GC_dont_gc!=0; } GC_API void**GC_CALL GC_new_free_list_inner(void) { void*result; GC_ASSERT(I_HOLD_LOCK()); result=GC_INTERNAL_MALLOC((MAXOBJGRANULES+1)*sizeof(ptr_t),PTRFREE); if (NULL==result)ABORT("Failed to allocate freelist for new kind"); BZERO(result,(MAXOBJGRANULES+1)*sizeof(ptr_t)); return (void**)result; } GC_API void**GC_CALL GC_new_free_list(void) { void**result; DCL_LOCK_STATE; LOCK(); result=GC_new_free_list_inner(); UNLOCK(); return result; } GC_API unsigned GC_CALL GC_new_kind_inner(void**fl,GC_word descr, int adjust,int clear) { unsigned result=GC_n_kinds; GC_ASSERT(NONNULL_ARG_NOT_NULL(fl)); GC_ASSERT(adjust==FALSE||adjust==TRUE); GC_ASSERT(clear==TRUE ||(descr==0&&adjust==FALSE&&clear==FALSE)); if (result < MAXOBJKINDS){ GC_ASSERT(result > 0); GC_n_kinds++; GC_obj_kinds[result].ok_freelist=fl; GC_obj_kinds[result].ok_reclaim_list=0; GC_obj_kinds[result].ok_descriptor=descr; GC_obj_kinds[result].ok_relocate_descr=adjust; GC_obj_kinds[result].ok_init=(GC_bool)clear; #ifdef ENABLE_DISCLAIM GC_obj_kinds[result].ok_mark_unconditionally=FALSE; GC_obj_kinds[result].ok_disclaim_proc=0; #endif } else { ABORT("Too many kinds"); } return result; } GC_API unsigned GC_CALL GC_new_kind(void**fl,GC_word descr,int adjust, int clear) { unsigned result; DCL_LOCK_STATE; LOCK(); result=GC_new_kind_inner(fl,descr,adjust,clear); UNLOCK(); return result; } GC_API unsigned GC_CALL GC_new_proc_inner(GC_mark_proc proc) { unsigned result=GC_n_mark_procs; if (result < MAX_MARK_PROCS){ GC_n_mark_procs++; GC_mark_procs[result]=proc; } else { ABORT("Too many mark procedures"); } return result; } GC_API unsigned GC_CALL GC_new_proc(GC_mark_proc proc) { unsigned result; DCL_LOCK_STATE; LOCK(); result=GC_new_proc_inner(proc); UNLOCK(); return result; } GC_API void*GC_CALL GC_call_with_alloc_lock(GC_fn_type fn,void*client_data) { void*result; DCL_LOCK_STATE; #ifdef THREADS LOCK(); #endif result=(*fn)(client_data); #ifdef THREADS UNLOCK(); #endif return(result); } GC_API void*GC_CALL GC_call_with_stack_base(GC_stack_base_func fn,void*arg) { struct GC_stack_base base; void*result; base.mem_base=(void*)&base; #ifdef IA64 base.reg_base=(void*)GC_save_regs_in_stack(); #endif result=fn(&base,arg); GC_noop1(COVERT_DATAFLOW(&base)); return result; } #ifndef THREADS GC_INNER ptr_t GC_blocked_sp=NULL; #ifdef IA64 STATIC ptr_t GC_blocked_register_sp=NULL; #endif GC_INNER struct GC_traced_stack_sect_s*GC_traced_stack_sect=NULL; GC_API void*GC_CALL GC_call_with_gc_active(GC_fn_type fn, void*client_data) { struct GC_traced_stack_sect_s stacksect; GC_ASSERT(GC_is_initialized); if ((word)GC_stackbottom HOTTER_THAN (word)(&stacksect)) GC_stackbottom=(ptr_t)COVERT_DATAFLOW(&stacksect); if (GC_blocked_sp==NULL){ client_data=fn(client_data); GC_noop1(COVERT_DATAFLOW(&stacksect)); return client_data; } stacksect.saved_stack_ptr=GC_blocked_sp; #ifdef IA64 stacksect.backing_store_end=GC_save_regs_in_stack(); stacksect.saved_backing_store_ptr=GC_blocked_register_sp; #endif stacksect.prev=GC_traced_stack_sect; GC_blocked_sp=NULL; GC_traced_stack_sect=&stacksect; client_data=fn(client_data); GC_ASSERT(GC_blocked_sp==NULL); GC_ASSERT(GC_traced_stack_sect==&stacksect); #if defined(CPPCHECK) GC_noop1((word)GC_traced_stack_sect - (word)GC_blocked_sp); #endif GC_traced_stack_sect=stacksect.prev; #ifdef IA64 GC_blocked_register_sp=stacksect.saved_backing_store_ptr; #endif GC_blocked_sp=stacksect.saved_stack_ptr; return client_data; } STATIC void GC_do_blocking_inner(ptr_t data,void*context GC_ATTR_UNUSED) { struct blocking_data*d=(struct blocking_data*)data; GC_ASSERT(GC_is_initialized); GC_ASSERT(GC_blocked_sp==NULL); #ifdef SPARC GC_blocked_sp=GC_save_regs_in_stack(); #else GC_blocked_sp=(ptr_t)&d; #endif #ifdef IA64 GC_blocked_register_sp=GC_save_regs_in_stack(); #endif d->client_data=(d->fn)(d->client_data); #ifdef SPARC GC_ASSERT(GC_blocked_sp!=NULL); #else GC_ASSERT(GC_blocked_sp==(ptr_t)(&d)); #endif #if defined(CPPCHECK) GC_noop1((word)GC_blocked_sp); #endif GC_blocked_sp=NULL; } GC_API void GC_CALL GC_set_stackbottom(void*gc_thread_handle, const struct GC_stack_base*sb) { GC_ASSERT(sb->mem_base!=NULL); GC_ASSERT(NULL==gc_thread_handle||&GC_stackbottom==gc_thread_handle); GC_ASSERT(NULL==GC_blocked_sp &&NULL==GC_traced_stack_sect); (void)gc_thread_handle; GC_stackbottom=(char*)sb->mem_base; #ifdef IA64 GC_register_stackbottom=(ptr_t)sb->reg_base; #endif } GC_API void*GC_CALL GC_get_my_stackbottom(struct GC_stack_base*sb) { GC_ASSERT(GC_is_initialized); sb->mem_base=GC_stackbottom; #ifdef IA64 sb->reg_base=GC_register_stackbottom; #endif return&GC_stackbottom; } #endif GC_API void*GC_CALL GC_do_blocking(GC_fn_type fn,void*client_data) { struct blocking_data my_data; my_data.fn=fn; my_data.client_data=client_data; GC_with_callee_saves_pushed(GC_do_blocking_inner,(ptr_t)(&my_data)); return my_data.client_data; } #if!defined(NO_DEBUGGING) GC_API void GC_CALL GC_dump(void) { DCL_LOCK_STATE; LOCK(); GC_dump_named(NULL); UNLOCK(); } GC_API void GC_CALL GC_dump_named(const char*name) { #ifndef NO_CLOCK CLOCK_TYPE current_time; GET_TIME(current_time); #endif if (name!=NULL){ GC_printf("***GC Dump %s\n",name); } else { GC_printf("***GC Dump collection #%lu\n",(unsigned long)GC_gc_no); } #ifndef NO_CLOCK GC_printf("Time since GC init:%lu ms\n", MS_TIME_DIFF(current_time,GC_init_time)); #endif GC_printf("\n***Static roots:\n"); GC_print_static_roots(); GC_printf("\n***Heap sections:\n"); GC_print_heap_sects(); GC_printf("\n***Free blocks:\n"); GC_print_hblkfreelist(); GC_printf("\n***Blocks in use:\n"); GC_print_block_list(); } #endif static void block_add_size(struct hblk*h,word pbytes) { hdr*hhdr=HDR(h); *(word*)pbytes+=(WORDS_TO_BYTES(hhdr->hb_sz)+(HBLKSIZE - 1)) &~(word)(HBLKSIZE - 1); } GC_API size_t GC_CALL GC_get_memory_use(void) { word bytes=0; DCL_LOCK_STATE; LOCK(); GC_apply_to_all_blocks(block_add_size,(word)(&bytes)); UNLOCK(); return (size_t)bytes; } GC_API GC_word GC_CALL GC_get_gc_no(void) { return GC_gc_no; } #ifdef THREADS GC_API int GC_CALL GC_get_parallel(void) { return GC_parallel; } GC_API void GC_CALL GC_alloc_lock(void) { DCL_LOCK_STATE; LOCK(); } GC_API void GC_CALL GC_alloc_unlock(void) { UNLOCK(); } GC_INNER GC_on_thread_event_proc GC_on_thread_event=0; GC_API void GC_CALL GC_set_on_thread_event(GC_on_thread_event_proc fn) { DCL_LOCK_STATE; LOCK(); GC_on_thread_event=fn; UNLOCK(); } GC_API GC_on_thread_event_proc GC_CALL GC_get_on_thread_event(void) { GC_on_thread_event_proc fn; DCL_LOCK_STATE; LOCK(); fn=GC_on_thread_event; UNLOCK(); return fn; } #endif GC_API void GC_CALL GC_set_oom_fn(GC_oom_func fn) { GC_ASSERT(NONNULL_ARG_NOT_NULL(fn)); DCL_LOCK_STATE; LOCK(); GC_oom_fn=fn; UNLOCK(); } GC_API GC_oom_func GC_CALL GC_get_oom_fn(void) { GC_oom_func fn; DCL_LOCK_STATE; LOCK(); fn=GC_oom_fn; UNLOCK(); return fn; } GC_API void GC_CALL GC_set_on_heap_resize(GC_on_heap_resize_proc fn) { DCL_LOCK_STATE; LOCK(); GC_on_heap_resize=fn; UNLOCK(); } GC_API GC_on_heap_resize_proc GC_CALL GC_get_on_heap_resize(void) { GC_on_heap_resize_proc fn; DCL_LOCK_STATE; LOCK(); fn=GC_on_heap_resize; UNLOCK(); return fn; } GC_API void GC_CALL GC_set_finalizer_notifier(GC_finalizer_notifier_proc fn) { DCL_LOCK_STATE; LOCK(); GC_finalizer_notifier=fn; UNLOCK(); } GC_API GC_finalizer_notifier_proc GC_CALL GC_get_finalizer_notifier(void) { GC_finalizer_notifier_proc fn; DCL_LOCK_STATE; LOCK(); fn=GC_finalizer_notifier; UNLOCK(); return fn; } GC_API void GC_CALL GC_set_find_leak(int value) { GC_find_leak=value; } GC_API int GC_CALL GC_get_find_leak(void) { return GC_find_leak; } GC_API void GC_CALL GC_set_all_interior_pointers(int value) { DCL_LOCK_STATE; GC_all_interior_pointers=value?1:0; if (GC_is_initialized){ LOCK(); GC_initialize_offsets(); if (!GC_all_interior_pointers) GC_bl_init_no_interiors(); UNLOCK(); } } GC_API int GC_CALL GC_get_all_interior_pointers(void) { return GC_all_interior_pointers; } GC_API void GC_CALL GC_set_finalize_on_demand(int value) { GC_ASSERT(value!=-1); GC_finalize_on_demand=value; } GC_API int GC_CALL GC_get_finalize_on_demand(void) { return GC_finalize_on_demand; } GC_API void GC_CALL GC_set_java_finalization(int value) { GC_ASSERT(value!=-1); GC_java_finalization=value; } GC_API int GC_CALL GC_get_java_finalization(void) { return GC_java_finalization; } GC_API void GC_CALL GC_set_dont_expand(int value) { GC_ASSERT(value!=-1); GC_dont_expand=value; } GC_API int GC_CALL GC_get_dont_expand(void) { return GC_dont_expand; } GC_API void GC_CALL GC_set_no_dls(int value) { GC_ASSERT(value!=-1); GC_no_dls=value; } GC_API int GC_CALL GC_get_no_dls(void) { return GC_no_dls; } GC_API void GC_CALL GC_set_non_gc_bytes(GC_word value) { GC_non_gc_bytes=value; } GC_API GC_word GC_CALL GC_get_non_gc_bytes(void) { return GC_non_gc_bytes; } GC_API void GC_CALL GC_set_free_space_divisor(GC_word value) { GC_ASSERT(value > 0); GC_free_space_divisor=value; } GC_API GC_word GC_CALL GC_get_free_space_divisor(void) { return GC_free_space_divisor; } GC_API void GC_CALL GC_set_max_retries(GC_word value) { GC_ASSERT((GC_signed_word)value!=-1); GC_max_retries=value; } GC_API GC_word GC_CALL GC_get_max_retries(void) { return GC_max_retries; } GC_API void GC_CALL GC_set_dont_precollect(int value) { GC_ASSERT(value!=-1); GC_dont_precollect=value; } GC_API int GC_CALL GC_get_dont_precollect(void) { return GC_dont_precollect; } GC_API void GC_CALL GC_set_full_freq(int value) { GC_ASSERT(value>=0); GC_full_freq=value; } GC_API int GC_CALL GC_get_full_freq(void) { return GC_full_freq; } GC_API void GC_CALL GC_set_time_limit(unsigned long value) { GC_ASSERT((long)value!=-1L); GC_time_limit=value; } GC_API unsigned long GC_CALL GC_get_time_limit(void) { return GC_time_limit; } GC_API void GC_CALL GC_set_force_unmap_on_gcollect(int value) { GC_force_unmap_on_gcollect=(GC_bool)value; } GC_API int GC_CALL GC_get_force_unmap_on_gcollect(void) { return (int)GC_force_unmap_on_gcollect; } GC_API void GC_CALL GC_abort_on_oom(void) { GC_err_printf("Insufficient memory for the allocation\n"); EXIT(); } #ifdef THREADS GC_API void GC_CALL GC_stop_world_external(void) { GC_ASSERT(GC_is_initialized); LOCK(); #ifdef THREAD_LOCAL_ALLOC GC_ASSERT(!GC_world_stopped); #endif STOP_WORLD(); #ifdef THREAD_LOCAL_ALLOC GC_world_stopped=TRUE; #endif } GC_API void GC_CALL GC_start_world_external(void) { #ifdef THREAD_LOCAL_ALLOC GC_ASSERT(GC_world_stopped); GC_world_stopped=FALSE; #else GC_ASSERT(GC_is_initialized); #endif START_WORLD(); UNLOCK(); } #endif #if!defined(OS2)&&!defined(PCR)&&!defined(AMIGA)&&!defined(MACOS)&&!defined(MSWINCE)&&!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PSP2)&&!defined(__CC_ARM) #include #if!defined(MSWIN32)&&!defined(MSWIN_XBOX1) #include #endif #endif #include #if defined(MSWINCE)||defined(SN_TARGET_PS3) #define SIGSEGV 0 #else #include #endif #if defined(UNIX_LIKE)||defined(CYGWIN32)||defined(NACL)||defined(SYMBIAN) #include #endif #if defined(LINUX)||defined(LINUX_STACKBOTTOM) #include #endif #ifdef AMIGA #define GC_AMIGA_DEF #if!defined(GC_AMIGA_DEF)&&!defined(GC_AMIGA_SB)&&!defined(GC_AMIGA_DS)&&!defined(GC_AMIGA_AM) #include #include #define GC_AMIGA_DEF #define GC_AMIGA_SB #define GC_AMIGA_DS #define GC_AMIGA_AM #endif #ifdef GC_AMIGA_DEF #ifndef __GNUC__ #include #endif #include #include #include #include #endif #ifdef GC_AMIGA_SB ptr_t GC_get_main_stack_base(void) { struct Process*proc=(struct Process*)SysBase->ThisTask; if (proc->pr_Task.tc_Node.ln_Type==NT_PROCESS &&proc->pr_CLI!=NULL){ return (char*)proc->pr_ReturnAddr+sizeof(ULONG); } else { return (char*)proc->pr_Task.tc_SPUpper; } } #endif #ifdef GC_AMIGA_DS void GC_register_data_segments(void) { struct Process*proc; struct CommandLineInterface*cli; BPTR myseglist; ULONG*data; #ifdef __GNUC__ ULONG dataSegSize; GC_bool found_segment=FALSE; extern char __data_size[]; dataSegSize=__data_size+8; #endif proc=(struct Process*)SysBase->ThisTask; myseglist=proc->pr_SegList; if (proc->pr_Task.tc_Node.ln_Type==NT_PROCESS){ if (proc->pr_CLI!=NULL){ cli=BADDR(proc->pr_CLI); myseglist=cli->cli_Module; } } else { ABORT("Not a Process."); } if (myseglist==NULL){ ABORT("Arrrgh.. can't find segments,aborting"); } for (data=(ULONG*)BADDR(myseglist);data!=NULL; data=(ULONG*)BADDR(data[0])){ if ((ULONG)GC_register_data_segments < (ULONG)(&data[1]) ||(ULONG)GC_register_data_segments > (ULONG)(&data[1]) +data[-1]){ #ifdef __GNUC__ if (dataSegSize==data[-1]){ found_segment=TRUE; } #endif GC_add_roots_inner((char*)&data[1], ((char*)&data[1])+data[-1],FALSE); } } #ifdef __GNUC__ if (!found_segment){ ABORT("Can`t find correct Segments.\nSolution:Use an newer version of ixemul.library"); } #endif } #endif #ifdef GC_AMIGA_AM #ifndef GC_AMIGA_FASTALLOC void*GC_amiga_allocwrapper(size_t size,void*(*AllocFunction)(size_t size2)){ return (*AllocFunction)(size); } void*(*GC_amiga_allocwrapper_do)(size_t size,void*(*AllocFunction)(size_t size2)) =GC_amiga_allocwrapper; #else void*GC_amiga_allocwrapper_firsttime(size_t size,void*(*AllocFunction)(size_t size2)); void*(*GC_amiga_allocwrapper_do)(size_t size,void*(*AllocFunction)(size_t size2)) =GC_amiga_allocwrapper_firsttime; struct GC_Amiga_AllocedMemoryHeader{ ULONG size; struct GC_Amiga_AllocedMemoryHeader*next; }; struct GC_Amiga_AllocedMemoryHeader*GC_AMIGAMEM=(struct GC_Amiga_AllocedMemoryHeader*)(int)~(NULL); ULONG GC_AMIGA_MEMF=MEMF_FAST|MEMF_CLEAR; #ifndef GC_AMIGA_ONLYFAST BOOL GC_amiga_dontalloc=FALSE; #endif #ifdef GC_AMIGA_PRINTSTATS int succ=0,succ2=0; int nsucc=0,nsucc2=0; int nullretries=0; int numcollects=0; int chipa=0; int allochip=0; int allocfast=0; int cur0=0; int cur1=0; int cur10=0; int cur50=0; int cur150=0; int cur151=0; int ncur0=0; int ncur1=0; int ncur10=0; int ncur50=0; int ncur150=0; int ncur151=0; #endif void GC_amiga_free_all_mem(void){ struct GC_Amiga_AllocedMemoryHeader*gc_am=(struct GC_Amiga_AllocedMemoryHeader*)(~(int)(GC_AMIGAMEM)); #ifdef GC_AMIGA_PRINTSTATS printf("\n\n" "%d bytes of chip-mem,and %d bytes of fast-mem where allocated from the OS.\n", allochip,allocfast ); printf( "%d bytes of chip-mem were returned from the GC_AMIGA_FASTALLOC supported allocating functions.\n", chipa ); printf("\n"); printf("GC_gcollect was called %d times to avoid returning NULL or start allocating with the MEMF_ANY flag.\n",numcollects); printf("%d of them was a success. (the others had to use allocation from the OS.)\n",nullretries); printf("\n"); printf("Succeeded forcing %d gc-allocations (%d bytes)of chip-mem to be fast-mem.\n",succ,succ2); printf("Failed forcing %d gc-allocations (%d bytes)of chip-mem to be fast-mem.\n",nsucc,nsucc2); printf("\n"); printf( "Number of retries before succeeding a chip->fast force:\n" "0:%d,1:%d,2-9:%d,10-49:%d,50-149:%d,>150:%d\n", cur0,cur1,cur10,cur50,cur150,cur151 ); printf( "Number of retries before giving up a chip->fast force:\n" "0:%d,1:%d,2-9:%d,10-49:%d,50-149:%d,>150:%d\n", ncur0,ncur1,ncur10,ncur50,ncur150,ncur151 ); #endif while(gc_am!=NULL){ struct GC_Amiga_AllocedMemoryHeader*temp=gc_am->next; FreeMem(gc_am,gc_am->size); gc_am=(struct GC_Amiga_AllocedMemoryHeader*)(~(int)(temp)); } } #ifndef GC_AMIGA_ONLYFAST char*chipmax; size_t latestsize; #endif #ifdef GC_AMIGA_FASTALLOC void*GC_amiga_get_mem(size_t size){ struct GC_Amiga_AllocedMemoryHeader*gc_am; #ifndef GC_AMIGA_ONLYFAST if(GC_amiga_dontalloc==TRUE){ return NULL; } if(GC_AMIGA_MEMF==(MEMF_ANY|MEMF_CLEAR)&&size>100000&&latestsize<50000)return NULL; #endif gc_am=AllocMem((ULONG)(size+sizeof(struct GC_Amiga_AllocedMemoryHeader)),GC_AMIGA_MEMF); if(gc_am==NULL)return NULL; gc_am->next=GC_AMIGAMEM; gc_am->size=size+sizeof(struct GC_Amiga_AllocedMemoryHeader); GC_AMIGAMEM=(struct GC_Amiga_AllocedMemoryHeader*)(~(int)(gc_am)); #ifdef GC_AMIGA_PRINTSTATS if((char*)gc_amchipmax||ret==NULL){ if(ret==NULL){ nsucc++; nsucc2+=size; if(rec==0)ncur0++; if(rec==1)ncur1++; if(rec>1&&rec<10)ncur10++; if(rec>=10&&rec<50)ncur50++; if(rec>=50&&rec<150)ncur150++; if(rec>=150)ncur151++; }else{ succ++; succ2+=size; if(rec==0)cur0++; if(rec==1)cur1++; if(rec>1&&rec<10)cur10++; if(rec>=10&&rec<50)cur50++; if(rec>=50&&rec<150)cur150++; if(rec>=150)cur151++; } } #endif if (((char*)ret)<=chipmax&&ret!=NULL&&(rec<(size>500000?9:size/5000))){ ret=GC_amiga_rec_alloc(size,AllocFunction,rec+1); } return ret; } #endif void*GC_amiga_allocwrapper_any(size_t size,void*(*AllocFunction)(size_t size2)){ void*ret; GC_amiga_dontalloc=TRUE; latestsize=size; ret=(*AllocFunction)(size); if(((char*)ret)<=chipmax){ if(ret==NULL){ #ifdef GC_AMIGA_GC if(!GC_dont_gc){ GC_gcollect(); #ifdef GC_AMIGA_PRINTSTATS numcollects++; #endif ret=(*AllocFunction)(size); } if(ret==NULL) #endif { GC_amiga_dontalloc=FALSE; ret=(*AllocFunction)(size); if(ret==NULL){ WARN("Out of Memory!Returning NIL!\n",0); } } #ifdef GC_AMIGA_PRINTSTATS else{ nullretries++; } if(ret!=NULL&&(char*)ret<=chipmax)chipa+=size; #endif } #ifdef GC_AMIGA_RETRY else{ void*ret2; if( AllocFunction!=GC_malloc_uncollectable #ifdef GC_ATOMIC_UNCOLLECTABLE &&AllocFunction!=GC_malloc_atomic_uncollectable #endif ){ ret2=GC_amiga_rec_alloc(size,AllocFunction,0); }else{ ret2=(*AllocFunction)(size); #ifdef GC_AMIGA_PRINTSTATS if((char*)ret2chipmax){ GC_free(ret); ret=ret2; }else{ GC_free(ret2); } } #endif } #if defined(CPPCHECK) if (GC_amiga_dontalloc) #endif GC_amiga_dontalloc=FALSE; return ret; } void (*GC_amiga_toany)(void)=NULL; void GC_amiga_set_toany(void (*func)(void)){ GC_amiga_toany=func; } #endif void*GC_amiga_allocwrapper_fast(size_t size,void*(*AllocFunction)(size_t size2)){ void*ret; ret=(*AllocFunction)(size); if(ret==NULL){ #ifdef GC_AMIGA_GC if(!GC_dont_gc){ GC_gcollect(); #ifdef GC_AMIGA_PRINTSTATS numcollects++; #endif ret=(*AllocFunction)(size); } if(ret==NULL) #endif { #ifndef GC_AMIGA_ONLYFAST GC_AMIGA_MEMF=MEMF_ANY|MEMF_CLEAR; if(GC_amiga_toany!=NULL)(*GC_amiga_toany)(); GC_amiga_allocwrapper_do=GC_amiga_allocwrapper_any; return GC_amiga_allocwrapper_any(size,AllocFunction); #endif } #ifdef GC_AMIGA_PRINTSTATS else{ nullretries++; } #endif } return ret; } void*GC_amiga_allocwrapper_firsttime(size_t size,void*(*AllocFunction)(size_t size2)){ atexit(&GC_amiga_free_all_mem); chipmax=(char*)SysBase->MaxLocMem; GC_amiga_allocwrapper_do=GC_amiga_allocwrapper_fast; return GC_amiga_allocwrapper_fast(size,AllocFunction); } #endif void*GC_amiga_realloc(void*old_object,size_t new_size_in_bytes){ #ifndef GC_AMIGA_FASTALLOC return GC_realloc(old_object,new_size_in_bytes); #else void*ret; latestsize=new_size_in_bytes; ret=GC_realloc(old_object,new_size_in_bytes); if(ret==NULL&&new_size_in_bytes!=0 &&GC_AMIGA_MEMF==(MEMF_FAST|MEMF_CLEAR)){ #ifdef GC_AMIGA_GC if(!GC_dont_gc){ GC_gcollect(); #ifdef GC_AMIGA_PRINTSTATS numcollects++; #endif ret=GC_realloc(old_object,new_size_in_bytes); } if(ret==NULL) #endif { #ifndef GC_AMIGA_ONLYFAST GC_AMIGA_MEMF=MEMF_ANY|MEMF_CLEAR; if(GC_amiga_toany!=NULL)(*GC_amiga_toany)(); GC_amiga_allocwrapper_do=GC_amiga_allocwrapper_any; ret=GC_realloc(old_object,new_size_in_bytes); #endif } #ifdef GC_AMIGA_PRINTSTATS else{ nullretries++; } #endif } if(ret==NULL&&new_size_in_bytes!=0){ WARN("Out of Memory!Returning NIL!\n",0); } #ifdef GC_AMIGA_PRINTSTATS if(((char*)ret) #endif #ifdef IRIX5 #include #include #endif #if defined(MMAP_SUPPORTED)||defined(ADD_HEAP_GUARD_PAGES) #if defined(USE_MUNMAP)&&!defined(USE_MMAP)&&!defined(CPPCHECK) #error Invalid config:USE_MUNMAP requires USE_MMAP #endif #include #include #include #include #endif #ifdef DARWIN #include #endif #ifdef DJGPP typedef long unsigned int caddr_t; #endif #ifdef PCR #include "mm/PCR_MM.h" #endif #if defined(GC_DARWIN_THREADS)&&defined(MPROTECT_VDB) #ifndef GC_DARWIN_STOP_WORLD_H #define GC_DARWIN_STOP_WORLD_H #if!defined(GC_DARWIN_THREADS) #error darwin_stop_world.h included without GC_DARWIN_THREADS defined #endif #include #include EXTERN_C_BEGIN struct thread_stop_info { mach_port_t mach_thread; ptr_t stack_ptr; }; #ifndef DARWIN_DONT_PARSE_STACK GC_INNER ptr_t GC_FindTopOfStack(unsigned long); #endif #ifdef MPROTECT_VDB GC_INNER void GC_mprotect_stop(void); GC_INNER void GC_mprotect_resume(void); #ifndef GC_NO_THREADS_DISCOVERY GC_INNER void GC_darwin_register_mach_handler_thread(mach_port_t thread); #endif #endif #if defined(PARALLEL_MARK)&&!defined(GC_NO_THREADS_DISCOVERY) GC_INNER GC_bool GC_is_mach_marker(thread_act_t); #endif EXTERN_C_END #endif #endif #if!defined(NO_EXECUTE_PERMISSION) STATIC GC_bool GC_pages_executable=TRUE; #else STATIC GC_bool GC_pages_executable=FALSE; #endif #define IGNORE_PAGES_EXECUTABLE 1 #ifdef NEED_PROC_MAPS STATIC ssize_t GC_repeat_read(int fd,char*buf,size_t count) { #define READ read size_t num_read=0; ASSERT_CANCEL_DISABLED(); while (num_read < count){ ssize_t result=READ(fd,buf+num_read,count - num_read); if (result < 0)return result; if (result==0)break; num_read+=result; } #undef READ return num_read; } #ifdef THREADS STATIC size_t GC_get_file_len(int f) { size_t total=0; ssize_t result; #define GET_FILE_LEN_BUF_SZ 500 char buf[GET_FILE_LEN_BUF_SZ]; do { result=read(f,buf,GET_FILE_LEN_BUF_SZ); if (result==-1)return 0; total+=result; } while (result > 0); return total; } STATIC size_t GC_get_maps_len(void) { int f=open("/proc/self/maps",O_RDONLY); size_t result; if (f < 0)return 0; result=GC_get_file_len(f); close(f); return result; } #endif GC_INNER char*GC_get_maps(void) { ssize_t result; static char*maps_buf=NULL; static size_t maps_buf_sz=1; size_t maps_size; #ifdef THREADS size_t old_maps_size=0; #endif GC_ASSERT(I_HOLD_LOCK()); #ifdef THREADS maps_size=GC_get_maps_len(); if (0==maps_size)return 0; #else maps_size=4000; #endif do { int f; while (maps_size>=maps_buf_sz){ #ifdef LINT2 GC_noop1((word)maps_buf); #else GC_scratch_recycle_no_gww(maps_buf,maps_buf_sz); #endif while (maps_size>=maps_buf_sz)maps_buf_sz*=2; maps_buf=GC_scratch_alloc(maps_buf_sz); #ifdef THREADS maps_size=GC_get_maps_len(); if (0==maps_size)return 0; #endif if (maps_buf==0)return 0; } GC_ASSERT(maps_buf_sz>=maps_size+1); f=open("/proc/self/maps",O_RDONLY); if (-1==f)return 0; #ifdef THREADS old_maps_size=maps_size; #endif maps_size=0; do { result=GC_repeat_read(f,maps_buf,maps_buf_sz-1); if (result<=0){ close(f); return 0; } maps_size+=result; } while ((size_t)result==maps_buf_sz-1); close(f); #ifdef THREADS if (maps_size > old_maps_size){ WARN("Unexpected asynchronous/proc/self/maps growth" " (to %" WARN_PRIdPTR " bytes)\n",maps_size); } #endif } while (maps_size>=maps_buf_sz #ifdef THREADS ||maps_size < old_maps_size #endif ); maps_buf[maps_size]='\0'; return maps_buf; } #if (defined(DYNAMIC_LOADING)&&defined(USE_PROC_FOR_LIBRARIES))||defined(IA64)||defined(INCLUDE_LINUX_THREAD_DESCR)||defined(REDIRECT_MALLOC) GC_INNER char*GC_parse_map_entry(char*buf_ptr,ptr_t*start,ptr_t*end, char**prot,unsigned int*maj_dev, char**mapping_name) { unsigned char*start_start,*end_start,*maj_dev_start; unsigned char*p; if (buf_ptr==NULL||*buf_ptr=='\0'){ return NULL; } p=(unsigned char*)buf_ptr; while (isspace(*p))++p; start_start=p; GC_ASSERT(isxdigit(*start_start)); *start=(ptr_t)strtoul((char*)start_start,(char**)&p,16); GC_ASSERT(*p=='-'); ++p; end_start=p; GC_ASSERT(isxdigit(*end_start)); *end=(ptr_t)strtoul((char*)end_start,(char**)&p,16); GC_ASSERT(isspace(*p)); while (isspace(*p))++p; GC_ASSERT(*p=='r'||*p=='-'); *prot=(char*)p; while (!isspace(*p))++p; while (isspace(*p))p++; GC_ASSERT(isxdigit(*p)); while (!isspace(*p))++p; while (isspace(*p))p++; maj_dev_start=p; GC_ASSERT(isxdigit(*maj_dev_start)); *maj_dev=strtoul((char*)maj_dev_start,NULL,16); if (mapping_name==0){ while (*p&&*p++!='\n'); } else { while (*p&&*p!='\n'&&*p!='/'&&*p!='[')p++; *mapping_name=(char*)p; while (*p&&*p++!='\n'); } return (char*)p; } #endif #if defined(IA64)||defined(INCLUDE_LINUX_THREAD_DESCR) GC_INNER GC_bool GC_enclosing_mapping(ptr_t addr,ptr_t*startp, ptr_t*endp) { char*prot; ptr_t my_start,my_end; unsigned int maj_dev; char*maps=GC_get_maps(); char*buf_ptr=maps; if (0==maps)return(FALSE); for (;;){ buf_ptr=GC_parse_map_entry(buf_ptr,&my_start,&my_end, &prot,&maj_dev,0); if (buf_ptr==NULL)return FALSE; if (prot[1]=='w'&&maj_dev==0){ if ((word)my_end > (word)addr&&(word)my_start<=(word)addr){ *startp=my_start; *endp=my_end; return TRUE; } } } return FALSE; } #endif #if defined(REDIRECT_MALLOC) GC_INNER GC_bool GC_text_mapping(char*nm,ptr_t*startp,ptr_t*endp) { size_t nm_len=strlen(nm); char*prot; char*map_path; ptr_t my_start,my_end; unsigned int maj_dev; char*maps=GC_get_maps(); char*buf_ptr=maps; if (0==maps)return(FALSE); for (;;){ buf_ptr=GC_parse_map_entry(buf_ptr,&my_start,&my_end, &prot,&maj_dev,&map_path); if (buf_ptr==NULL)return FALSE; if (prot[0]=='r'&&prot[1]=='-'&&prot[2]=='x'){ char*p=map_path; while (*p!='\0'&&*p!='\n'&&*p!=' '&&*p!='\t')++p; while (*p!='/'&&(word)p>=(word)map_path)--p; ++p; if (strncmp(nm,p,nm_len)==0){ *startp=my_start; *endp=my_end; return TRUE; } } } return FALSE; } #endif #ifdef IA64 static ptr_t backing_store_base_from_proc(void) { ptr_t my_start,my_end; if (!GC_enclosing_mapping(GC_save_regs_in_stack(),&my_start,&my_end)){ GC_COND_LOG_PRINTF("Failed to find backing store base from/proc\n"); return 0; } return my_start; } #endif #endif #if defined(SEARCH_FOR_DATA_START) #if defined(LINUX)||defined(HURD) EXTERN_C_BEGIN #pragma weak __data_start #pragma weak data_start extern int __data_start[],data_start[]; EXTERN_C_END #endif ptr_t GC_data_start=NULL; GC_INNER void GC_init_linux_data_start(void) { ptr_t data_end=DATAEND; #if (defined(LINUX)||defined(HURD))&&defined(USE_PROG_DATA_START) if (COVERT_DATAFLOW(__data_start)!=0){ GC_data_start=(ptr_t)(__data_start); } else { GC_data_start=(ptr_t)(data_start); } if (COVERT_DATAFLOW(GC_data_start)!=0){ if ((word)GC_data_start > (word)data_end) ABORT_ARG2("Wrong __data_start/_end pair", ":%p .. %p",(void*)GC_data_start,(void*)data_end); return; } #ifdef DEBUG_ADD_DEL_ROOTS GC_log_printf("__data_start not provided\n"); #endif #endif if (GC_no_dls){ GC_data_start=data_end; return; } GC_data_start=(ptr_t)GC_find_limit(data_end,FALSE); } #endif #ifdef ECOS #ifndef ECOS_GC_MEMORY_SIZE #define ECOS_GC_MEMORY_SIZE (448*1024) #endif static char ecos_gc_memory[ECOS_GC_MEMORY_SIZE]; static char*ecos_gc_brk=ecos_gc_memory; static void*tiny_sbrk(ptrdiff_t increment) { void*p=ecos_gc_brk; ecos_gc_brk+=increment; if ((word)ecos_gc_brk > (word)(ecos_gc_memory+sizeof(ecos_gc_memory))){ ecos_gc_brk-=increment; return NULL; } return p; } #define sbrk tiny_sbrk #endif #if defined(NETBSD)&&defined(__ELF__) ptr_t GC_data_start=NULL; EXTERN_C_BEGIN extern char**environ; EXTERN_C_END GC_INNER void GC_init_netbsd_elf(void) { GC_data_start=(ptr_t)GC_find_limit(&environ,FALSE); } #endif #if defined(ADDRESS_SANITIZER)&&(defined(UNIX_LIKE)||defined(NEED_FIND_LIMIT)||defined(MPROTECT_VDB))&&!defined(CUSTOM_ASAN_DEF_OPTIONS) GC_API const char*__asan_default_options(void) { return "allow_user_segv_handler=1"; } #endif #ifdef OPENBSD static struct sigaction old_segv_act; STATIC JMP_BUF GC_jmp_buf_openbsd; STATIC void GC_fault_handler_openbsd(int sig GC_ATTR_UNUSED) { LONGJMP(GC_jmp_buf_openbsd,1); } #ifdef GC_OPENBSD_UTHREADS #include EXTERN_C_BEGIN extern sigset_t __syscall(quad_t,...); EXTERN_C_END STATIC ptr_t GC_find_limit_openbsd(ptr_t p,ptr_t bound) { static volatile ptr_t result; struct sigaction act; word pgsz=(word)sysconf(_SC_PAGESIZE); GC_ASSERT((word)bound>=pgsz); GC_ASSERT(I_HOLD_LOCK()); act.sa_handler=GC_fault_handler_openbsd; sigemptyset(&act.sa_mask); act.sa_flags=SA_NODEFER|SA_RESTART; sigaction(SIGSEGV,&act,&old_segv_act); if (SETJMP(GC_jmp_buf_openbsd)==0){ result=(ptr_t)((word)p&~(pgsz-1)); for (;;){ if ((word)result>=(word)bound - pgsz){ result=bound; break; } result+=pgsz; GC_noop1((word)(*result)); } } #ifdef THREADS __syscall(SYS_sigprocmask,SIG_UNBLOCK,sigmask(SIGPROF)); #endif sigaction(SIGSEGV,&old_segv_act,0); return(result); } #endif static volatile int firstpass; STATIC ptr_t GC_skip_hole_openbsd(ptr_t p,ptr_t bound) { static volatile ptr_t result; struct sigaction act; word pgsz=(word)sysconf(_SC_PAGESIZE); GC_ASSERT((word)bound>=pgsz); GC_ASSERT(I_HOLD_LOCK()); act.sa_handler=GC_fault_handler_openbsd; sigemptyset(&act.sa_mask); act.sa_flags=SA_NODEFER|SA_RESTART; sigaction(SIGSEGV,&act,&old_segv_act); firstpass=1; result=(ptr_t)((word)p&~(pgsz-1)); if (SETJMP(GC_jmp_buf_openbsd)!=0||firstpass){ firstpass=0; if ((word)result>=(word)bound - pgsz){ result=bound; } else { result+=pgsz; GC_noop1((word)(*result)); } } sigaction(SIGSEGV,&old_segv_act,0); return(result); } #endif #ifdef OS2 #include #if!defined(__IBMC__)&&!defined(__WATCOMC__) struct exe_hdr { unsigned short magic_number; unsigned short padding[29]; long new_exe_offset; }; #define E_MAGIC(x)(x).magic_number #define EMAGIC 0x5A4D #define E_LFANEW(x)(x).new_exe_offset struct e32_exe { unsigned char magic_number[2]; unsigned char byte_order; unsigned char word_order; unsigned long exe_format_level; unsigned short cpu; unsigned short os; unsigned long padding1[13]; unsigned long object_table_offset; unsigned long object_count; unsigned long padding2[31]; }; #define E32_MAGIC1(x)(x).magic_number[0] #define E32MAGIC1 'L' #define E32_MAGIC2(x)(x).magic_number[1] #define E32MAGIC2 'X' #define E32_BORDER(x)(x).byte_order #define E32LEBO 0 #define E32_WORDER(x)(x).word_order #define E32LEWO 0 #define E32_CPU(x)(x).cpu #define E32CPU286 1 #define E32_OBJTAB(x)(x).object_table_offset #define E32_OBJCNT(x)(x).object_count struct o32_obj { unsigned long size; unsigned long base; unsigned long flags; unsigned long pagemap; unsigned long mapsize; unsigned long reserved; }; #define O32_FLAGS(x)(x).flags #define OBJREAD 0x0001L #define OBJWRITE 0x0002L #define OBJINVALID 0x0080L #define O32_SIZE(x)(x).size #define O32_BASE(x)(x).base #else #ifndef WORD #define WORD unsigned short #endif #ifndef DWORD #define DWORD unsigned long #endif #define EXE386 1 #include #include #endif #define INCL_DOSEXCEPTIONS #define INCL_DOSPROCESS #define INCL_DOSERRORS #define INCL_DOSMODULEMGR #define INCL_DOSMEMMGR #include #endif GC_INNER size_t GC_page_size=0; #if defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32) #ifndef VER_PLATFORM_WIN32_CE #define VER_PLATFORM_WIN32_CE 3 #endif #if defined(MSWINCE)&&defined(THREADS) GC_INNER GC_bool GC_dont_query_stack_min=FALSE; #endif GC_INNER SYSTEM_INFO GC_sysinfo; GC_INNER void GC_setpagesize(void) { GetSystemInfo(&GC_sysinfo); #if defined(CYGWIN32)&&(defined(MPROTECT_VDB)||defined(USE_MUNMAP)) GC_page_size=(size_t)GC_sysinfo.dwAllocationGranularity; GC_ASSERT(GC_page_size>=(size_t)GC_sysinfo.dwPageSize); #else GC_page_size=(size_t)GC_sysinfo.dwPageSize; #endif #if defined(MSWINCE)&&!defined(_WIN32_WCE_EMULATION) { OSVERSIONINFO verInfo; verInfo.dwOSVersionInfoSize=sizeof(OSVERSIONINFO); if (!GetVersionEx(&verInfo)) ABORT("GetVersionEx failed"); if (verInfo.dwPlatformId==VER_PLATFORM_WIN32_CE&& verInfo.dwMajorVersion < 6){ GC_sysinfo.lpMaximumApplicationAddress=(LPVOID)((word)32<<20); #ifdef THREADS if (verInfo.dwMajorVersion < 5) GC_dont_query_stack_min=TRUE; #endif } } #endif } #ifndef CYGWIN32 #define is_writable(prot)((prot)==PAGE_READWRITE||(prot)==PAGE_WRITECOPY||(prot)==PAGE_EXECUTE_READWRITE||(prot)==PAGE_EXECUTE_WRITECOPY) STATIC word GC_get_writable_length(ptr_t p,ptr_t*base) { MEMORY_BASIC_INFORMATION buf; word result; word protect; result=VirtualQuery(p,&buf,sizeof(buf)); if (result!=sizeof(buf))ABORT("Weird VirtualQuery result"); if (base!=0)*base=(ptr_t)(buf.AllocationBase); protect=(buf.Protect&~(PAGE_GUARD|PAGE_NOCACHE)); if (!is_writable(protect)){ return(0); } if (buf.State!=MEM_COMMIT)return(0); return(buf.RegionSize); } GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base*sb) { ptr_t trunc_sp; word size; if (!GC_page_size)GC_setpagesize(); trunc_sp=(ptr_t)((word)GC_approx_sp()&~(GC_page_size - 1)); size=GC_get_writable_length(trunc_sp,0); GC_ASSERT(size!=0); sb->mem_base=trunc_sp+size; return GC_SUCCESS; } #else GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base*sb) { #ifdef X86_64 sb->mem_base=((NT_TIB*)NtCurrentTeb())->StackBase; #else void*_tlsbase; __asm__ ("movl %%fs:4,%0" :"=r" (_tlsbase)); sb->mem_base=_tlsbase; #endif return GC_SUCCESS; } #endif #define HAVE_GET_STACK_BASE #else GC_INNER void GC_setpagesize(void) { #if defined(MPROTECT_VDB)||defined(PROC_VDB)||defined(USE_MMAP) GC_page_size=(size_t)GETPAGESIZE(); #if!defined(CPPCHECK) if (0==GC_page_size) ABORT("getpagesize failed"); #endif #else GC_page_size=HBLKSIZE; #endif } #endif #ifdef HAIKU #include GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base*sb) { thread_info th; get_thread_info(find_thread(NULL),&th); sb->mem_base=th.stack_end; return GC_SUCCESS; } #define HAVE_GET_STACK_BASE #endif #ifdef OS2 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base*sb) { PTIB ptib; PPIB ppib; if (DosGetInfoBlocks(&ptib,&ppib)!=NO_ERROR){ WARN("DosGetInfoBlocks failed\n",0); return GC_UNIMPLEMENTED; } sb->mem_base=ptib->tib_pstacklimit; return GC_SUCCESS; } #define HAVE_GET_STACK_BASE #endif #ifdef AMIGA #define GC_AMIGA_SB #undef GC_AMIGA_SB #define GET_MAIN_STACKBASE_SPECIAL #endif #if defined(NEED_FIND_LIMIT)||defined(UNIX_LIKE) typedef void (*GC_fault_handler_t)(int); #if defined(SUNOS5SIGS)||defined(IRIX5)||defined(OSF1)||defined(HAIKU)||defined(HURD)||defined(FREEBSD)||defined(NETBSD) static struct sigaction old_segv_act; #if defined(_sigargs)||defined(HURD)||defined(NETBSD)||defined(FREEBSD) static struct sigaction old_bus_act; #endif #else static GC_fault_handler_t old_segv_handler; #ifdef HAVE_SIGBUS static GC_fault_handler_t old_bus_handler; #endif #endif GC_INNER void GC_set_and_save_fault_handler(GC_fault_handler_t h) { #if defined(SUNOS5SIGS)||defined(IRIX5)||defined(OSF1)||defined(HAIKU)||defined(HURD)||defined(FREEBSD)||defined(NETBSD)||defined(OPENBSD) struct sigaction act; act.sa_handler=h; #ifdef SIGACTION_FLAGS_NODEFER_HACK act.sa_flags=SA_RESTART|SA_NODEFER; #else act.sa_flags=SA_RESTART; #endif (void)sigemptyset(&act.sa_mask); #ifdef GC_IRIX_THREADS (void)sigaction(SIGSEGV,0,&old_segv_act); (void)sigaction(SIGSEGV,&act,0); #else (void)sigaction(SIGSEGV,&act,&old_segv_act); #if defined(IRIX5)&&defined(_sigargs)||defined(HURD)||defined(NETBSD)||defined(FREEBSD) (void)sigaction(SIGBUS,&act,&old_bus_act); #endif #endif #else old_segv_handler=signal(SIGSEGV,h); #ifdef HAVE_SIGBUS old_bus_handler=signal(SIGBUS,h); #endif #endif #if defined(CPPCHECK)&&defined(ADDRESS_SANITIZER) GC_noop1((word)&__asan_default_options); #endif } #endif #if defined(NEED_FIND_LIMIT)||(defined(USE_PROC_FOR_LIBRARIES)&&defined(THREADS)) #define MIN_PAGE_SIZE 256 GC_INNER JMP_BUF GC_jmp_buf; STATIC void GC_fault_handler(int sig GC_ATTR_UNUSED) { LONGJMP(GC_jmp_buf,1); } GC_INNER void GC_setup_temporary_fault_handler(void) { GC_ASSERT(I_HOLD_LOCK()); GC_set_and_save_fault_handler(GC_fault_handler); } GC_INNER void GC_reset_fault_handler(void) { #if defined(SUNOS5SIGS)||defined(IRIX5)||defined(OSF1)||defined(HAIKU)||defined(HURD)||defined(FREEBSD)||defined(NETBSD)||defined(OPENBSD) (void)sigaction(SIGSEGV,&old_segv_act,0); #if defined(IRIX5)&&defined(_sigargs)||defined(HURD)||defined(NETBSD) (void)sigaction(SIGBUS,&old_bus_act,0); #endif #else (void)signal(SIGSEGV,old_segv_handler); #ifdef HAVE_SIGBUS (void)signal(SIGBUS,old_bus_handler); #endif #endif } GC_ATTR_NO_SANITIZE_ADDR STATIC ptr_t GC_find_limit_with_bound(ptr_t p,GC_bool up,ptr_t bound) { static volatile ptr_t result; GC_ASSERT(up?(word)bound>=MIN_PAGE_SIZE :(word)bound<=~(word)MIN_PAGE_SIZE); GC_ASSERT(I_HOLD_LOCK()); GC_setup_temporary_fault_handler(); if (SETJMP(GC_jmp_buf)==0){ result=(ptr_t)(((word)(p)) &~(MIN_PAGE_SIZE-1)); for (;;){ if (up){ if ((word)result>=(word)bound - MIN_PAGE_SIZE){ result=bound; break; } result+=MIN_PAGE_SIZE; } else { if ((word)result<=(word)bound+MIN_PAGE_SIZE){ result=bound - MIN_PAGE_SIZE; break; } result-=MIN_PAGE_SIZE; } GC_noop1((word)(*result)); } } GC_reset_fault_handler(); if (!up){ result+=MIN_PAGE_SIZE; } return(result); } void*GC_find_limit(void*p,int up) { return GC_find_limit_with_bound((ptr_t)p,(GC_bool)up, up?(ptr_t)GC_WORD_MAX:0); } #endif #ifdef HPUX_MAIN_STACKBOTTOM #include #include STATIC ptr_t GC_hpux_main_stack_base(void) { struct pst_vm_status vm_status; int i=0; while (pstat_getprocvm(&vm_status,sizeof(vm_status),0,i++)==1){ if (vm_status.pst_type==PS_STACK) return (ptr_t)vm_status.pst_vaddr; } #ifdef STACK_GROWS_UP return (ptr_t)GC_find_limit(GC_approx_sp(),FALSE); #else return (ptr_t)GC_find_limit(GC_approx_sp(),TRUE); #endif } #endif #ifdef HPUX_STACKBOTTOM #include #include GC_INNER ptr_t GC_get_register_stack_base(void) { struct pst_vm_status vm_status; int i=0; while (pstat_getprocvm(&vm_status,sizeof(vm_status),0,i++)==1){ if (vm_status.pst_type==PS_RSESTACK){ return (ptr_t)vm_status.pst_vaddr; } } return (ptr_t)(((word)GC_stackbottom - BACKING_STORE_DISPLACEMENT - 1) &~(BACKING_STORE_ALIGNMENT - 1)); } #endif #ifdef LINUX_STACKBOTTOM #include #include #define STAT_SKIP 27 #ifdef USE_LIBC_PRIVATES EXTERN_C_BEGIN #pragma weak __libc_stack_end extern ptr_t __libc_stack_end; #ifdef IA64 #pragma weak __libc_ia64_register_backing_store_base extern ptr_t __libc_ia64_register_backing_store_base; #endif EXTERN_C_END #endif #ifdef IA64 GC_INNER ptr_t GC_get_register_stack_base(void) { ptr_t result; #ifdef USE_LIBC_PRIVATES if (0!=&__libc_ia64_register_backing_store_base &&0!=__libc_ia64_register_backing_store_base){ return __libc_ia64_register_backing_store_base; } #endif result=backing_store_base_from_proc(); if (0==result){ result=(ptr_t)GC_find_limit(GC_save_regs_in_stack(),FALSE); } return result; } #endif STATIC ptr_t GC_linux_main_stack_base(void) { #ifndef STAT_READ #define STAT_BUF_SIZE 4096 #define STAT_READ read #endif char stat_buf[STAT_BUF_SIZE]; int f; word result; int i,buf_offset=0,len; #ifdef USE_LIBC_PRIVATES if (0!=&__libc_stack_end&&0!=__libc_stack_end){ #if defined(IA64) if (((word)__libc_stack_end&0xfff)+0x10 < 0x1000){ return __libc_stack_end+0x10; } #elif defined(SPARC) if (__libc_stack_end!=(ptr_t)(unsigned long)0x1) return __libc_stack_end; #else return __libc_stack_end; #endif } #endif f=open("/proc/self/stat",O_RDONLY); if (f < 0) ABORT("Couldn't read/proc/self/stat"); len=STAT_READ(f,stat_buf,STAT_BUF_SIZE); close(f); for (i=0;i < STAT_SKIP;++i){ while (buf_offset < len&&isspace(stat_buf[buf_offset++])){ } while (buf_offset < len&&!isspace(stat_buf[buf_offset++])){ } } while (buf_offset < len&&isspace(stat_buf[buf_offset])){ buf_offset++; } for (i=0;buf_offset+i < len;i++){ if (!isdigit(stat_buf[buf_offset+i]))break; } if (buf_offset+i>=len)ABORT("Could not parse/proc/self/stat"); stat_buf[buf_offset+i]='\0'; result=(word)STRTOULL(&stat_buf[buf_offset],NULL,10); if (result < 0x100000||(result&(sizeof(word)- 1))!=0) ABORT("Absurd stack bottom value"); return (ptr_t)result; } #endif #ifdef FREEBSD_STACKBOTTOM #include #include #include STATIC ptr_t GC_freebsd_main_stack_base(void) { int nm[2]={CTL_KERN,KERN_USRSTACK}; ptr_t base; size_t len=sizeof(ptr_t); int r=sysctl(nm,2,&base,&len,NULL,0); if (r)ABORT("Error getting main stack base"); return base; } #endif #if defined(ECOS)||defined(NOSYS) ptr_t GC_get_main_stack_base(void) { return STACKBOTTOM; } #define GET_MAIN_STACKBASE_SPECIAL #elif defined(SYMBIAN) EXTERN_C_BEGIN extern int GC_get_main_symbian_stack_base(void); EXTERN_C_END ptr_t GC_get_main_stack_base(void) { return (ptr_t)GC_get_main_symbian_stack_base(); } #define GET_MAIN_STACKBASE_SPECIAL #elif defined(__EMSCRIPTEN__) #include static void*emscripten_stack_base; static void scan_stack_cb(void*begin,void*end) { (void)begin; emscripten_stack_base=end; } ptr_t GC_get_main_stack_base(void) { emscripten_scan_stack(scan_stack_cb); return (ptr_t)emscripten_stack_base; } #define GET_MAIN_STACKBASE_SPECIAL #elif!defined(AMIGA)&&!defined(HAIKU)&&!defined(OS2)&&!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(CYGWIN32)&&!defined(GC_OPENBSD_THREADS)&&(!defined(GC_SOLARIS_THREADS)||defined(_STRICT_STDC)) #if (defined(HAVE_PTHREAD_ATTR_GET_NP)||defined(HAVE_PTHREAD_GETATTR_NP))&&(defined(THREADS)||defined(USE_GET_STACKBASE_FOR_MAIN)) #include #ifdef HAVE_PTHREAD_NP_H #include #endif #elif defined(DARWIN)&&!defined(NO_PTHREAD_GET_STACKADDR_NP) #include #undef STACKBOTTOM #define STACKBOTTOM (ptr_t)pthread_get_stackaddr_np(pthread_self()) #endif ptr_t GC_get_main_stack_base(void) { ptr_t result; #if (defined(HAVE_PTHREAD_ATTR_GET_NP)||defined(HAVE_PTHREAD_GETATTR_NP))&&(defined(USE_GET_STACKBASE_FOR_MAIN)||(defined(THREADS)&&!defined(REDIRECT_MALLOC))) pthread_attr_t attr; void*stackaddr; size_t size; #ifdef HAVE_PTHREAD_ATTR_GET_NP if (pthread_attr_init(&attr)==0 &&(pthread_attr_get_np(pthread_self(),&attr)==0 ?TRUE:(pthread_attr_destroy(&attr),FALSE))) #else if (pthread_getattr_np(pthread_self(),&attr)==0) #endif { if (pthread_attr_getstack(&attr,&stackaddr,&size)==0 &&stackaddr!=NULL){ (void)pthread_attr_destroy(&attr); #ifdef STACK_GROWS_DOWN stackaddr=(char*)stackaddr+size; #endif return (ptr_t)stackaddr; } (void)pthread_attr_destroy(&attr); } WARN("pthread_getattr_np or pthread_attr_getstack failed" " for main thread\n",0); #endif #ifdef STACKBOTTOM result=STACKBOTTOM; #else #ifdef HEURISTIC1 #define STACKBOTTOM_ALIGNMENT_M1 ((word)STACK_GRAN - 1) #ifdef STACK_GROWS_DOWN result=(ptr_t)(((word)GC_approx_sp()+STACKBOTTOM_ALIGNMENT_M1) &~STACKBOTTOM_ALIGNMENT_M1); #else result=(ptr_t)((word)GC_approx_sp()&~STACKBOTTOM_ALIGNMENT_M1); #endif #elif defined(HPUX_MAIN_STACKBOTTOM) result=GC_hpux_main_stack_base(); #elif defined(LINUX_STACKBOTTOM) result=GC_linux_main_stack_base(); #elif defined(FREEBSD_STACKBOTTOM) result=GC_freebsd_main_stack_base(); #elif defined(HEURISTIC2) { ptr_t sp=GC_approx_sp(); #ifdef STACK_GROWS_DOWN result=(ptr_t)GC_find_limit(sp,TRUE); #if defined(HEURISTIC2_LIMIT)&&!defined(CPPCHECK) if ((word)result > (word)HEURISTIC2_LIMIT &&(word)sp < (word)HEURISTIC2_LIMIT){ result=HEURISTIC2_LIMIT; } #endif #else result=(ptr_t)GC_find_limit(sp,FALSE); #if defined(HEURISTIC2_LIMIT)&&!defined(CPPCHECK) if ((word)result < (word)HEURISTIC2_LIMIT &&(word)sp > (word)HEURISTIC2_LIMIT){ result=HEURISTIC2_LIMIT; } #endif #endif } #elif defined(STACK_NOT_SCANNED)||defined(CPPCHECK) result=NULL; #else #error None of HEURISTIC*and*STACKBOTTOM defined! #endif #if defined(STACK_GROWS_DOWN)&&!defined(CPPCHECK) if (result==0) result=(ptr_t)(signed_word)(-sizeof(ptr_t)); #endif #endif #if!defined(CPPCHECK) GC_ASSERT((word)GC_approx_sp()HOTTER_THAN (word)result); #endif return(result); } #define GET_MAIN_STACKBASE_SPECIAL #endif #if (defined(HAVE_PTHREAD_ATTR_GET_NP)||defined(HAVE_PTHREAD_GETATTR_NP))&&defined(THREADS)&&!defined(HAVE_GET_STACK_BASE) #include #ifdef HAVE_PTHREAD_NP_H #include #endif GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base*b) { pthread_attr_t attr; size_t size; #ifdef IA64 DCL_LOCK_STATE; #endif #ifdef HAVE_PTHREAD_ATTR_GET_NP if (pthread_attr_init(&attr)!=0) ABORT("pthread_attr_init failed"); if (pthread_attr_get_np(pthread_self(),&attr)!=0){ WARN("pthread_attr_get_np failed\n",0); (void)pthread_attr_destroy(&attr); return GC_UNIMPLEMENTED; } #else if (pthread_getattr_np(pthread_self(),&attr)!=0){ WARN("pthread_getattr_np failed\n",0); return GC_UNIMPLEMENTED; } #endif if (pthread_attr_getstack(&attr,&(b->mem_base),&size)!=0){ ABORT("pthread_attr_getstack failed"); } (void)pthread_attr_destroy(&attr); #ifdef STACK_GROWS_DOWN b->mem_base=(char*)(b->mem_base)+size; #endif #ifdef IA64 LOCK(); { IF_CANCEL(int cancel_state;) ptr_t bsp; ptr_t next_stack; DISABLE_CANCEL(cancel_state); bsp=GC_save_regs_in_stack(); next_stack=GC_greatest_stack_base_below(bsp); if (0==next_stack){ b->reg_base=GC_find_limit(bsp,FALSE); } else { b->reg_base=GC_find_limit_with_bound(bsp,FALSE,next_stack); } RESTORE_CANCEL(cancel_state); } UNLOCK(); #endif return GC_SUCCESS; } #define HAVE_GET_STACK_BASE #endif #if defined(GC_DARWIN_THREADS)&&!defined(NO_PTHREAD_GET_STACKADDR_NP) #include GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base*b) { b->mem_base=pthread_get_stackaddr_np(pthread_self()); GC_ASSERT((word)GC_approx_sp()HOTTER_THAN (word)b->mem_base); return GC_SUCCESS; } #define HAVE_GET_STACK_BASE #endif #ifdef GC_OPENBSD_THREADS #include #include #include GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base*sb) { stack_t stack; if (pthread_stackseg_np(pthread_self(),&stack)) ABORT("pthread_stackseg_np(self)failed"); sb->mem_base=stack.ss_sp; return GC_SUCCESS; } #define HAVE_GET_STACK_BASE #endif #if defined(GC_SOLARIS_THREADS)&&!defined(_STRICT_STDC) #include #include #include static pthread_t stackbase_main_self=0; static void*stackbase_main_ss_sp=NULL; GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base*b) { stack_t s; pthread_t self=pthread_self(); if (self==stackbase_main_self) { b->mem_base=stackbase_main_ss_sp; GC_ASSERT(b->mem_base!=NULL); return GC_SUCCESS; } if (thr_stksegment(&s)){ ABORT("thr_stksegment failed"); } GC_ASSERT((word)GC_approx_sp()HOTTER_THAN (word)s.ss_sp); if (!stackbase_main_self&&thr_main()!=0) { stackbase_main_ss_sp=s.ss_sp; stackbase_main_self=self; } b->mem_base=s.ss_sp; return GC_SUCCESS; } #define HAVE_GET_STACK_BASE #endif #ifdef GC_RTEMS_PTHREADS GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base*sb) { sb->mem_base=rtems_get_stack_bottom(); return GC_SUCCESS; } #define HAVE_GET_STACK_BASE #endif #ifndef HAVE_GET_STACK_BASE #ifdef NEED_FIND_LIMIT GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base*b) { IF_CANCEL(int cancel_state;) DCL_LOCK_STATE; LOCK(); DISABLE_CANCEL(cancel_state); #ifdef STACK_GROWS_DOWN b->mem_base=GC_find_limit(GC_approx_sp(),TRUE); #ifdef IA64 b->reg_base=GC_find_limit(GC_save_regs_in_stack(),FALSE); #endif #else b->mem_base=GC_find_limit(GC_approx_sp(),FALSE); #endif RESTORE_CANCEL(cancel_state); UNLOCK(); return GC_SUCCESS; } #else GC_API int GC_CALL GC_get_stack_base( struct GC_stack_base*b GC_ATTR_UNUSED) { #if defined(GET_MAIN_STACKBASE_SPECIAL)&&!defined(THREADS)&&!defined(IA64) b->mem_base=GC_get_main_stack_base(); return GC_SUCCESS; #else return GC_UNIMPLEMENTED; #endif } #endif #endif #ifndef GET_MAIN_STACKBASE_SPECIAL ptr_t GC_get_main_stack_base(void) { struct GC_stack_base sb; if (GC_get_stack_base(&sb)!=GC_SUCCESS) ABORT("GC_get_stack_base failed"); GC_ASSERT((word)GC_approx_sp()HOTTER_THAN (word)sb.mem_base); return (ptr_t)sb.mem_base; } #endif #ifdef OS2 void GC_register_data_segments(void) { PTIB ptib; PPIB ppib; HMODULE module_handle; #define PBUFSIZ 512 UCHAR path[PBUFSIZ]; FILE*myexefile; struct exe_hdr hdrdos; struct e32_exe hdr386; struct o32_obj seg; int nsegs; #if defined(CPPCHECK) hdrdos.padding[0]=0; hdr386.exe_format_level=0; hdr386.os=0; hdr386.padding1[0]=0; hdr386.padding2[0]=0; seg.pagemap=0; seg.mapsize=0; seg.reserved=0; #endif if (DosGetInfoBlocks(&ptib,&ppib)!=NO_ERROR){ ABORT("DosGetInfoBlocks failed"); } module_handle=ppib->pib_hmte; if (DosQueryModuleName(module_handle,PBUFSIZ,path)!=NO_ERROR){ ABORT("DosQueryModuleName failed"); } myexefile=fopen(path,"rb"); if (myexefile==0){ ABORT_ARG1("Failed to open executable",":%s",path); } if (fread((char*)(&hdrdos),1,sizeof(hdrdos),myexefile) < sizeof(hdrdos)){ ABORT_ARG1("Could not read MSDOS header"," from:%s",path); } if (E_MAGIC(hdrdos)!=EMAGIC){ ABORT_ARG1("Bad DOS magic number"," in file:%s",path); } if (fseek(myexefile,E_LFANEW(hdrdos),SEEK_SET)!=0){ ABORT_ARG1("Bad DOS magic number"," in file:%s",path); } if (fread((char*)(&hdr386),1,sizeof(hdr386),myexefile) < sizeof(hdr386)){ ABORT_ARG1("Could not read OS/2 header"," from:%s",path); } if (E32_MAGIC1(hdr386)!=E32MAGIC1||E32_MAGIC2(hdr386)!=E32MAGIC2){ ABORT_ARG1("Bad OS/2 magic number"," in file:%s",path); } if (E32_BORDER(hdr386)!=E32LEBO||E32_WORDER(hdr386)!=E32LEWO){ ABORT_ARG1("Bad byte order in executable"," file:%s",path); } if (E32_CPU(hdr386)==E32CPU286){ ABORT_ARG1("GC cannot handle 80286 executables",":%s",path); } if (fseek(myexefile,E_LFANEW(hdrdos)+E32_OBJTAB(hdr386), SEEK_SET)!=0){ ABORT_ARG1("Seek to object table failed"," in file:%s",path); } for (nsegs=E32_OBJCNT(hdr386);nsegs > 0;nsegs--){ int flags; if (fread((char*)(&seg),1,sizeof(seg),myexefile)< sizeof(seg)){ ABORT_ARG1("Could not read obj table entry"," from file:%s",path); } flags=O32_FLAGS(seg); if (!(flags&OBJWRITE))continue; if (!(flags&OBJREAD))continue; if (flags&OBJINVALID){ GC_err_printf("Object with invalid pages?\n"); continue; } GC_add_roots_inner((ptr_t)O32_BASE(seg), (ptr_t)(O32_BASE(seg)+O32_SIZE(seg)),FALSE); } (void)fclose(myexefile); } #else #if defined(GWW_VDB) #ifndef MEM_WRITE_WATCH #define MEM_WRITE_WATCH 0x200000 #endif #ifndef WRITE_WATCH_FLAG_RESET #define WRITE_WATCH_FLAG_RESET 1 #endif #define GC_ULONG_PTR word typedef UINT (WINAPI*GetWriteWatch_type)( DWORD,PVOID,GC_ULONG_PTR, PVOID*,GC_ULONG_PTR*,PULONG); static FARPROC GetWriteWatch_func; static DWORD GetWriteWatch_alloc_flag; #define GC_GWW_AVAILABLE()(GetWriteWatch_func!=0) static void detect_GetWriteWatch(void) { static GC_bool done; HMODULE hK32; if (done) return; #if defined(MPROTECT_VDB) { char*str=GETENV("GC_USE_GETWRITEWATCH"); #if defined(GC_PREFER_MPROTECT_VDB) if (str==NULL||(*str=='0'&&*(str+1)=='\0')){ done=TRUE; return; } #else if (str!=NULL&&*str=='0'&&*(str+1)=='\0'){ done=TRUE; return; } #endif } #endif #ifdef MSWINRT_FLAVOR { MEMORY_BASIC_INFORMATION memInfo; SIZE_T result=VirtualQuery(GetProcAddress, &memInfo,sizeof(memInfo)); if (result!=sizeof(memInfo)) ABORT("Weird VirtualQuery result"); hK32=(HMODULE)memInfo.AllocationBase; } #else hK32=GetModuleHandle(TEXT("kernel32.dll")); #endif if (hK32!=(HMODULE)0&& (GetWriteWatch_func=GetProcAddress(hK32,"GetWriteWatch"))!=0){ void*page; GC_ASSERT(GC_page_size!=0); page=VirtualAlloc(NULL,GC_page_size,MEM_WRITE_WATCH|MEM_RESERVE, PAGE_READWRITE); if (page!=NULL){ PVOID pages[16]; GC_ULONG_PTR count=16; DWORD page_size; if ((*(GetWriteWatch_type)(word)GetWriteWatch_func)( WRITE_WATCH_FLAG_RESET,page, GC_page_size,pages,&count, &page_size)!=0){ GetWriteWatch_func=0; } else { GetWriteWatch_alloc_flag=MEM_WRITE_WATCH; } VirtualFree(page,0,MEM_RELEASE); } else { GetWriteWatch_func=0; } } #ifndef SMALL_CONFIG if (!GetWriteWatch_func){ GC_COND_LOG_PRINTF("Did not find a usable GetWriteWatch()\n"); } else { GC_COND_LOG_PRINTF("Using GetWriteWatch()\n"); } #endif done=TRUE; } #else #define GetWriteWatch_alloc_flag 0 #endif #if defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32) #ifdef MSWIN32 GC_INNER GC_bool GC_no_win32_dlls=FALSE; GC_INNER GC_bool GC_wnt=FALSE; GC_INNER void GC_init_win32(void) { #if defined(_WIN64)||(defined(_MSC_VER)&&_MSC_VER>=1800) GC_wnt=TRUE; #else DWORD v=GetVersion(); GC_wnt=!(v&0x80000000); GC_no_win32_dlls|=((!GC_wnt)&&(v&0xff)<=3); #endif #ifdef USE_MUNMAP if (GC_no_win32_dlls){ GC_unmap_threshold=0; } #endif } STATIC ptr_t GC_least_described_address(ptr_t start) { MEMORY_BASIC_INFORMATION buf; LPVOID limit=GC_sysinfo.lpMinimumApplicationAddress; ptr_t p=(ptr_t)((word)start&~(GC_page_size - 1)); GC_ASSERT(GC_page_size!=0); for (;;){ size_t result; LPVOID q=(LPVOID)(p - GC_page_size); if ((word)q > (word)p||(word)q < (word)limit)break; result=VirtualQuery(q,&buf,sizeof(buf)); if (result!=sizeof(buf)||buf.AllocationBase==0)break; p=(ptr_t)(buf.AllocationBase); } return p; } #endif #if defined(USE_WINALLOC)&&!defined(REDIRECT_MALLOC) STATIC size_t GC_max_root_size=100000; STATIC struct GC_malloc_heap_list { void*allocation_base; struct GC_malloc_heap_list*next; }*GC_malloc_heap_l=0; STATIC GC_bool GC_is_malloc_heap_base(void*p) { struct GC_malloc_heap_list*q=GC_malloc_heap_l; while (0!=q){ if (q->allocation_base==p)return TRUE; q=q->next; } return FALSE; } STATIC void*GC_get_allocation_base(void*p) { MEMORY_BASIC_INFORMATION buf; size_t result=VirtualQuery(p,&buf,sizeof(buf)); if (result!=sizeof(buf)){ ABORT("Weird VirtualQuery result"); } return buf.AllocationBase; } GC_INNER void GC_add_current_malloc_heap(void) { struct GC_malloc_heap_list*new_l=(struct GC_malloc_heap_list*) malloc(sizeof(struct GC_malloc_heap_list)); void*candidate; if (NULL==new_l)return; candidate=GC_get_allocation_base(new_l); if (GC_is_malloc_heap_base(candidate)){ size_t req_size=10000; do { void*p=malloc(req_size); if (0==p){ free(new_l); return; } candidate=GC_get_allocation_base(p); free(p); req_size*=2; } while (GC_is_malloc_heap_base(candidate) &&req_size < GC_max_root_size/10&&req_size < 500000); if (GC_is_malloc_heap_base(candidate)){ free(new_l); return; } } GC_COND_LOG_PRINTF("Found new system malloc AllocationBase at %p\n", candidate); new_l->allocation_base=candidate; new_l->next=GC_malloc_heap_l; GC_malloc_heap_l=new_l; } STATIC void GC_free_malloc_heap_list(void) { struct GC_malloc_heap_list*q=GC_malloc_heap_l; GC_malloc_heap_l=NULL; while (q!=NULL){ struct GC_malloc_heap_list*next=q->next; free(q); q=next; } } #endif GC_INNER GC_bool GC_is_heap_base(void*p) { int i; #if defined(USE_WINALLOC)&&!defined(REDIRECT_MALLOC) if (GC_root_size > GC_max_root_size) GC_max_root_size=GC_root_size; if (GC_is_malloc_heap_base(p)) return TRUE; #endif for (i=0;i < (int)GC_n_heap_bases;i++){ if (GC_heap_bases[i]==p)return TRUE; } return FALSE; } #ifdef MSWIN32 STATIC void GC_register_root_section(ptr_t static_root) { MEMORY_BASIC_INFORMATION buf; LPVOID p; char*base; char*limit; if (!GC_no_win32_dlls)return; p=base=limit=GC_least_described_address(static_root); while ((word)p < (word)GC_sysinfo.lpMaximumApplicationAddress){ size_t result=VirtualQuery(p,&buf,sizeof(buf)); char*new_limit; DWORD protect; if (result!=sizeof(buf)||buf.AllocationBase==0 ||GC_is_heap_base(buf.AllocationBase))break; new_limit=(char*)p+buf.RegionSize; protect=buf.Protect; if (buf.State==MEM_COMMIT &&is_writable(protect)){ if ((char*)p==limit){ limit=new_limit; } else { if (base!=limit)GC_add_roots_inner(base,limit,FALSE); base=(char*)p; limit=new_limit; } } if ((word)p > (word)new_limit)break; p=(LPVOID)new_limit; } if (base!=limit)GC_add_roots_inner(base,limit,FALSE); } #endif void GC_register_data_segments(void) { #ifdef MSWIN32 GC_register_root_section((ptr_t)&GC_pages_executable); #endif } #else #if (defined(SVR4)||defined(AIX)||defined(DGUX)||(defined(LINUX)&&defined(SPARC)))&&!defined(PCR) ptr_t GC_SysVGetDataStart(size_t max_page_size,ptr_t etext_addr) { word text_end=((word)(etext_addr)+sizeof(word)- 1) &~(word)(sizeof(word)- 1); word next_page=((text_end+(word)max_page_size - 1) &~((word)max_page_size - 1)); word page_offset=(text_end&((word)max_page_size - 1)); volatile ptr_t result=(char*)(next_page+page_offset); GC_setup_temporary_fault_handler(); if (SETJMP(GC_jmp_buf)==0){ #ifdef AO_HAVE_fetch_and_add volatile AO_t zero=0; (void)AO_fetch_and_add((volatile AO_t*)result,zero); #else char v=*result; #if defined(CPPCHECK) GC_noop1((word)&v); #endif *result=v; #endif GC_reset_fault_handler(); } else { GC_reset_fault_handler(); result=(char*)GC_find_limit(DATAEND,FALSE); } return ( ptr_t)result; } #endif #ifdef DATASTART_USES_BSDGETDATASTART GC_INNER ptr_t GC_FreeBSDGetDataStart(size_t max_page_size, ptr_t etext_addr) { word text_end=((word)(etext_addr)+sizeof(word)- 1) &~(word)(sizeof(word)- 1); volatile word next_page=(text_end+(word)max_page_size - 1) &~((word)max_page_size - 1); volatile ptr_t result=(ptr_t)text_end; GC_setup_temporary_fault_handler(); if (SETJMP(GC_jmp_buf)==0){ for (;next_page < (word)DATAEND;next_page+=(word)max_page_size) *(volatile char*)next_page; GC_reset_fault_handler(); } else { GC_reset_fault_handler(); result=(ptr_t)GC_find_limit(DATAEND,FALSE); } return(result); } #endif #ifdef AMIGA #define GC_AMIGA_DS #undef GC_AMIGA_DS #elif defined(OPENBSD) void GC_register_data_segments(void) { ptr_t region_start=DATASTART; if ((word)region_start - 1U>=(word)DATAEND) ABORT_ARG2("Wrong DATASTART/END pair", ":%p .. %p",(void*)region_start,(void*)DATAEND); for (;;){ #ifdef GC_OPENBSD_UTHREADS ptr_t region_end=GC_find_limit_openbsd(region_start,DATAEND); #else ptr_t region_end=GC_find_limit_with_bound(region_start,TRUE,DATAEND); #endif GC_add_roots_inner(region_start,region_end,FALSE); if ((word)region_end>=(word)DATAEND) break; region_start=GC_skip_hole_openbsd(region_end,DATAEND); } } #else #if!defined(PCR)&&!defined(MACOS)&&defined(REDIRECT_MALLOC)&&defined(GC_SOLARIS_THREADS) EXTERN_C_BEGIN extern caddr_t sbrk(int); EXTERN_C_END #endif void GC_register_data_segments(void) { #if!defined(PCR)&&!defined(MACOS) #if defined(REDIRECT_MALLOC)&&defined(GC_SOLARIS_THREADS) GC_ASSERT(DATASTART); { ptr_t p=(ptr_t)sbrk(0); if ((word)DATASTART < (word)p) GC_add_roots_inner(DATASTART,p,FALSE); } #else if ((word)DATASTART - 1U>=(word)DATAEND){ ABORT_ARG2("Wrong DATASTART/END pair", ":%p .. %p",(void*)DATASTART,(void*)DATAEND); } GC_add_roots_inner(DATASTART,DATAEND,FALSE); #ifdef GC_HAVE_DATAREGION2 if ((word)DATASTART2 - 1U>=(word)DATAEND2) ABORT_ARG2("Wrong DATASTART/END2 pair", ":%p .. %p",(void*)DATASTART2,(void*)DATAEND2); GC_add_roots_inner(DATASTART2,DATAEND2,FALSE); #endif #endif #endif #if defined(MACOS) { #if defined(THINK_C) extern void*GC_MacGetDataStart(void); GC_add_roots_inner((ptr_t)GC_MacGetDataStart(), (ptr_t)LMGetCurrentA5(),FALSE); #else #if defined(__MWERKS__) #if!__POWERPC__ extern void*GC_MacGetDataStart(void); #if __option(far_data) extern void*GC_MacGetDataEnd(void); #endif GC_add_roots_inner((ptr_t)GC_MacGetDataStart(), (ptr_t)LMGetCurrentA5(),FALSE); #if __option(far_data) GC_add_roots_inner((ptr_t)LMGetCurrentA5(), (ptr_t)GC_MacGetDataEnd(),FALSE); #endif #else extern char __data_start__[],__data_end__[]; GC_add_roots_inner((ptr_t)&__data_start__, (ptr_t)&__data_end__,FALSE); #endif #endif #endif } #endif } #endif #endif #endif #if!defined(OS2)&&!defined(PCR)&&!defined(AMIGA)&&!defined(USE_WINALLOC)&&!defined(MACOS)&&!defined(DOS4GW)&&!defined(NINTENDO_SWITCH)&&!defined(NONSTOP)&&!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PS3)&&!defined(SN_TARGET_PSP2)&&!defined(RTEMS)&&!defined(__CC_ARM) #define SBRK_ARG_T ptrdiff_t #if defined(MMAP_SUPPORTED) #ifdef USE_MMAP_FIXED #define GC_MMAP_FLAGS MAP_FIXED|MAP_PRIVATE #else #define GC_MMAP_FLAGS MAP_PRIVATE #endif #ifdef USE_MMAP_ANON #define zero_fd -1 #if defined(MAP_ANONYMOUS)&&!defined(CPPCHECK) #define OPT_MAP_ANON MAP_ANONYMOUS #else #define OPT_MAP_ANON MAP_ANON #endif #else static int zero_fd=-1; #define OPT_MAP_ANON 0 #endif #ifndef MSWIN_XBOX1 #if defined(SYMBIAN)&&!defined(USE_MMAP_ANON) EXTERN_C_BEGIN extern char*GC_get_private_path_and_zero_file(void); EXTERN_C_END #endif STATIC ptr_t GC_unix_mmap_get_mem(size_t bytes) { void*result; static ptr_t last_addr=HEAP_START; #ifndef USE_MMAP_ANON static GC_bool initialized=FALSE; if (!EXPECT(initialized,TRUE)){ #ifdef SYMBIAN char*path=GC_get_private_path_and_zero_file(); if (path!=NULL){ zero_fd=open(path,O_RDWR|O_CREAT,0644); free(path); } #else zero_fd=open("/dev/zero",O_RDONLY); #endif if (zero_fd==-1) ABORT("Could not open/dev/zero"); if (fcntl(zero_fd,F_SETFD,FD_CLOEXEC)==-1) WARN("Could not set FD_CLOEXEC for/dev/zero\n",0); initialized=TRUE; } #endif GC_ASSERT(GC_page_size!=0); if (bytes&(GC_page_size - 1))ABORT("Bad GET_MEM arg"); result=mmap(last_addr,bytes,(PROT_READ|PROT_WRITE) |(GC_pages_executable?PROT_EXEC:0), GC_MMAP_FLAGS|OPT_MAP_ANON,zero_fd,0); #undef IGNORE_PAGES_EXECUTABLE if (EXPECT(MAP_FAILED==result,FALSE)){ if (HEAP_START==last_addr&&GC_pages_executable&&EACCES==errno) ABORT("Cannot allocate executable pages"); return NULL; } last_addr=(ptr_t)(((word)result+bytes+GC_page_size - 1) &~(GC_page_size - 1)); #if!defined(LINUX) if (last_addr==0){ munmap(result,~GC_page_size - (size_t)result+1); return GC_unix_mmap_get_mem(bytes); } #else GC_ASSERT(last_addr!=0); #endif if (((word)result % HBLKSIZE)!=0) ABORT( "GC_unix_get_mem:Memory returned by mmap is not aligned to HBLKSIZE."); return((ptr_t)result); } #endif #endif #if defined(USE_MMAP) ptr_t GC_unix_get_mem(size_t bytes) { return GC_unix_mmap_get_mem(bytes); } #else STATIC ptr_t GC_unix_sbrk_get_mem(size_t bytes) { ptr_t result; #ifdef IRIX5 __LOCK_MALLOC(); #endif { ptr_t cur_brk=(ptr_t)sbrk(0); SBRK_ARG_T lsbs=(word)cur_brk&(GC_page_size-1); GC_ASSERT(GC_page_size!=0); if ((SBRK_ARG_T)bytes < 0){ result=0; goto out; } if (lsbs!=0){ if((ptr_t)sbrk((SBRK_ARG_T)GC_page_size - lsbs)==(ptr_t)(-1)){ result=0; goto out; } } #ifdef ADD_HEAP_GUARD_PAGES { ptr_t guard=(ptr_t)sbrk((SBRK_ARG_T)GC_page_size); if (mprotect(guard,GC_page_size,PROT_NONE)!=0) ABORT("ADD_HEAP_GUARD_PAGES:mprotect failed"); } #endif result=(ptr_t)sbrk((SBRK_ARG_T)bytes); if (result==(ptr_t)(-1))result=0; } out: #ifdef IRIX5 __UNLOCK_MALLOC(); #endif return(result); } ptr_t GC_unix_get_mem(size_t bytes) { #if defined(MMAP_SUPPORTED) static GC_bool sbrk_failed=FALSE; ptr_t result=0; if (GC_pages_executable){ return GC_unix_mmap_get_mem(bytes); } if (!sbrk_failed)result=GC_unix_sbrk_get_mem(bytes); if (0==result){ sbrk_failed=TRUE; result=GC_unix_mmap_get_mem(bytes); } if (0==result){ result=GC_unix_sbrk_get_mem(bytes); } return result; #else return GC_unix_sbrk_get_mem(bytes); #endif } #endif #endif #ifdef OS2 void*os2_alloc(size_t bytes) { void*result; if (DosAllocMem(&result,bytes,(PAG_READ|PAG_WRITE|PAG_COMMIT) |(GC_pages_executable?PAG_EXECUTE:0)) !=NO_ERROR){ return(0); } if (result==0)return(os2_alloc(bytes)); return(result); } #endif #ifdef MSWIN_XBOX1 ptr_t GC_durango_get_mem(size_t bytes) { if (0==bytes)return NULL; return (ptr_t)VirtualAlloc(NULL,bytes,MEM_COMMIT|MEM_TOP_DOWN, PAGE_READWRITE); } #elif defined(MSWINCE) ptr_t GC_wince_get_mem(size_t bytes) { ptr_t result=0; word i; GC_ASSERT(GC_page_size!=0); bytes=ROUNDUP_PAGESIZE(bytes); for (i=0;i < GC_n_heap_bases;i++){ if (((word)(-(signed_word)GC_heap_lengths[i]) &(GC_sysinfo.dwAllocationGranularity-1)) >=bytes){ result=GC_heap_bases[i]+GC_heap_lengths[i]; break; } } if (i==GC_n_heap_bases){ size_t res_bytes= SIZET_SAT_ADD(bytes,(size_t)GC_sysinfo.dwAllocationGranularity-1) &~((size_t)GC_sysinfo.dwAllocationGranularity-1); result=(ptr_t)VirtualAlloc(NULL,res_bytes, MEM_RESERVE|MEM_TOP_DOWN, GC_pages_executable?PAGE_EXECUTE_READWRITE: PAGE_READWRITE); if (HBLKDISPL(result)!=0)ABORT("Bad VirtualAlloc result"); if (GC_n_heap_bases>=MAX_HEAP_SECTS)ABORT("Too many heap sections"); if (result==NULL)return NULL; GC_heap_bases[GC_n_heap_bases]=result; GC_heap_lengths[GC_n_heap_bases]=0; GC_n_heap_bases++; } result=(ptr_t)VirtualAlloc(result,bytes,MEM_COMMIT, GC_pages_executable?PAGE_EXECUTE_READWRITE: PAGE_READWRITE); #undef IGNORE_PAGES_EXECUTABLE if (result!=NULL){ if (HBLKDISPL(result)!=0)ABORT("Bad VirtualAlloc result"); GC_heap_lengths[i]+=bytes; } return(result); } #elif (defined(USE_WINALLOC)&&!defined(MSWIN_XBOX1))||defined(CYGWIN32) #ifdef USE_GLOBAL_ALLOC #define GLOBAL_ALLOC_TEST 1 #else #define GLOBAL_ALLOC_TEST GC_no_win32_dlls #endif #if (defined(GC_USE_MEM_TOP_DOWN)&&defined(USE_WINALLOC))||defined(CPPCHECK) DWORD GC_mem_top_down=MEM_TOP_DOWN; #else #define GC_mem_top_down 0 #endif ptr_t GC_win32_get_mem(size_t bytes) { ptr_t result; #ifndef USE_WINALLOC result=GC_unix_get_mem(bytes); #else #if defined(MSWIN32)&&!defined(MSWINRT_FLAVOR) if (GLOBAL_ALLOC_TEST){ result=(ptr_t)GlobalAlloc(0,SIZET_SAT_ADD(bytes,HBLKSIZE)); result=(ptr_t)(((word)result+HBLKSIZE - 1) &~(word)(HBLKSIZE - 1)); } else #endif { #ifdef MPROTECT_VDB #ifdef GWW_VDB #define VIRTUAL_ALLOC_PAD (GC_GWW_AVAILABLE()?0:1) #else #define VIRTUAL_ALLOC_PAD 1 #endif #else #define VIRTUAL_ALLOC_PAD 0 #endif result=(ptr_t)VirtualAlloc(NULL, SIZET_SAT_ADD(bytes,VIRTUAL_ALLOC_PAD), GetWriteWatch_alloc_flag |(MEM_COMMIT|MEM_RESERVE) |GC_mem_top_down, GC_pages_executable?PAGE_EXECUTE_READWRITE: PAGE_READWRITE); #undef IGNORE_PAGES_EXECUTABLE } #endif if (HBLKDISPL(result)!=0)ABORT("Bad VirtualAlloc result"); if (GC_n_heap_bases>=MAX_HEAP_SECTS)ABORT("Too many heap sections"); if (0!=result)GC_heap_bases[GC_n_heap_bases++]=result; return(result); } #endif #if defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32)||defined(MSWIN_XBOX1) GC_API void GC_CALL GC_win32_free_heap(void) { #if defined(USE_WINALLOC)&&!defined(REDIRECT_MALLOC)&&!defined(MSWIN_XBOX1) GC_free_malloc_heap_list(); #endif #if (defined(USE_WINALLOC)&&!defined(MSWIN_XBOX1)&&!defined(MSWINCE))||defined(CYGWIN32) #ifndef MSWINRT_FLAVOR #ifndef CYGWIN32 if (GLOBAL_ALLOC_TEST) #endif { while (GC_n_heap_bases--> 0){ #ifdef CYGWIN32 #else GlobalFree(GC_heap_bases[GC_n_heap_bases]); #endif GC_heap_bases[GC_n_heap_bases]=0; } return; } #endif #ifndef CYGWIN32 while (GC_n_heap_bases > 0){ VirtualFree(GC_heap_bases[--GC_n_heap_bases],0,MEM_RELEASE); GC_heap_bases[GC_n_heap_bases]=0; } #endif #endif } #endif #ifdef AMIGA #define GC_AMIGA_AM #undef GC_AMIGA_AM #endif #if defined(HAIKU) #include ptr_t GC_haiku_get_mem(size_t bytes) { void*mem; GC_ASSERT(GC_page_size!=0); if (posix_memalign(&mem,GC_page_size,bytes)==0) return mem; return NULL; } #endif #ifdef USE_MUNMAP #if!defined(NN_PLATFORM_CTR)&&!defined(MSWIN32)&&!defined(MSWINCE)&&!defined(MSWIN_XBOX1) #include #ifdef SN_TARGET_PS3 #include #else #include #endif #include #include #endif STATIC ptr_t GC_unmap_start(ptr_t start,size_t bytes) { ptr_t result=(ptr_t)(((word)start+GC_page_size - 1) &~(GC_page_size - 1)); GC_ASSERT(GC_page_size!=0); if ((word)(result+GC_page_size)> (word)(start+bytes))return 0; return result; } GC_INNER void GC_unmap(ptr_t start,size_t bytes) { ptr_t start_addr=GC_unmap_start(start,bytes); ptr_t end_addr=GC_unmap_end(start,bytes); word len=end_addr - start_addr; if (0==start_addr)return; #ifdef USE_WINALLOC while (len!=0){ MEMORY_BASIC_INFORMATION mem_info; word free_len; if (VirtualQuery(start_addr,&mem_info,sizeof(mem_info)) !=sizeof(mem_info)) ABORT("Weird VirtualQuery result"); free_len=(len < mem_info.RegionSize)?len:mem_info.RegionSize; if (!VirtualFree(start_addr,free_len,MEM_DECOMMIT)) ABORT("VirtualFree failed"); GC_unmapped_bytes+=free_len; start_addr+=free_len; len-=free_len; } #elif defined(SN_TARGET_PS3) ps3_free_mem(start_addr,len); #else { #if defined(AIX)||defined(CYGWIN32)||defined(HAIKU)||defined(HPUX) if (mprotect(start_addr,len,PROT_NONE)) ABORT("mprotect(PROT_NONE)failed"); #elif defined(__EMSCRIPTEN__) #else void*result=mmap(start_addr,len,PROT_NONE, MAP_PRIVATE|MAP_FIXED|OPT_MAP_ANON, zero_fd,0); if (result!=(void*)start_addr) ABORT("mmap(PROT_NONE)failed"); #if defined(CPPCHECK)||defined(LINT2) GC_noop1((word)result); #endif #endif } GC_unmapped_bytes+=len; #endif } GC_INNER void GC_remap(ptr_t start,size_t bytes) { ptr_t start_addr=GC_unmap_start(start,bytes); ptr_t end_addr=GC_unmap_end(start,bytes); word len=end_addr - start_addr; if (0==start_addr)return; #ifdef USE_WINALLOC while (len!=0){ MEMORY_BASIC_INFORMATION mem_info; word alloc_len; ptr_t result; if (VirtualQuery(start_addr,&mem_info,sizeof(mem_info)) !=sizeof(mem_info)) ABORT("Weird VirtualQuery result"); alloc_len=(len < mem_info.RegionSize)?len:mem_info.RegionSize; result=(ptr_t)VirtualAlloc(start_addr,alloc_len,MEM_COMMIT, GC_pages_executable ?PAGE_EXECUTE_READWRITE :PAGE_READWRITE); if (result!=start_addr){ if (GetLastError()==ERROR_NOT_ENOUGH_MEMORY|| GetLastError()==ERROR_OUTOFMEMORY){ ABORT("Not enough memory to process remapping"); } else { ABORT("VirtualAlloc remapping failed"); } } #ifdef LINT2 GC_noop1((word)result); #endif GC_unmapped_bytes-=alloc_len; start_addr+=alloc_len; len-=alloc_len; } #else { #if defined(NACL)||defined(NETBSD) void*result=mmap(start_addr,len,(PROT_READ|PROT_WRITE) |(GC_pages_executable?PROT_EXEC:0), MAP_PRIVATE|MAP_FIXED|OPT_MAP_ANON, zero_fd,0); if (result!=(void*)start_addr) ABORT("mmap as mprotect failed"); #if defined(CPPCHECK)||defined(LINT2) GC_noop1((word)result); #endif #else if (mprotect(start_addr,len,(PROT_READ|PROT_WRITE) |(GC_pages_executable?PROT_EXEC:0))!=0){ ABORT_ARG3("mprotect remapping failed", " at %p (length %lu),errcode=%d", (void*)start_addr,(unsigned long)len,errno); } #endif } #undef IGNORE_PAGES_EXECUTABLE GC_unmapped_bytes-=len; #endif } GC_INNER void GC_unmap_gap(ptr_t start1,size_t bytes1,ptr_t start2, size_t bytes2) { ptr_t start1_addr=GC_unmap_start(start1,bytes1); ptr_t end1_addr=GC_unmap_end(start1,bytes1); ptr_t start2_addr=GC_unmap_start(start2,bytes2); ptr_t start_addr=end1_addr; ptr_t end_addr=start2_addr; size_t len; GC_ASSERT(start1+bytes1==start2); if (0==start1_addr)start_addr=GC_unmap_start(start1,bytes1+bytes2); if (0==start2_addr)end_addr=GC_unmap_end(start1,bytes1+bytes2); if (0==start_addr)return; len=end_addr - start_addr; #ifdef USE_WINALLOC while (len!=0){ MEMORY_BASIC_INFORMATION mem_info; word free_len; if (VirtualQuery(start_addr,&mem_info,sizeof(mem_info)) !=sizeof(mem_info)) ABORT("Weird VirtualQuery result"); free_len=(len < mem_info.RegionSize)?len:mem_info.RegionSize; if (!VirtualFree(start_addr,free_len,MEM_DECOMMIT)) ABORT("VirtualFree failed"); GC_unmapped_bytes+=free_len; start_addr+=free_len; len-=free_len; } #else if (len!=0){ #if defined(AIX)||defined(CYGWIN32)||defined(HAIKU)||defined(HPUX) if (mprotect(start_addr,len,PROT_NONE)) ABORT("mprotect(PROT_NONE)failed"); #else void*result=mmap(start_addr,len,PROT_NONE, MAP_PRIVATE|MAP_FIXED|OPT_MAP_ANON, zero_fd,0); if (result!=(void*)start_addr) ABORT("mmap(PROT_NONE)failed"); #if defined(CPPCHECK)||defined(LINT2) GC_noop1((word)result); #endif #endif GC_unmapped_bytes+=len; } #endif } #endif #ifndef THREADS #if defined(__EMSCRIPTEN__) static void scan_regs_cb(void*begin,void*end) { GC_push_all_stack((ptr_t)begin,(ptr_t)end); } STATIC void GC_CALLBACK GC_default_push_other_roots(void) { emscripten_scan_registers(scan_regs_cb); } #else #define GC_default_push_other_roots 0 #endif #else #ifdef PCR PCR_ERes GC_push_thread_stack(PCR_Th_T*t,PCR_Any dummy) { struct PCR_ThCtl_TInfoRep info; PCR_ERes result; info.ti_stkLow=info.ti_stkHi=0; result=PCR_ThCtl_GetInfo(t,&info); GC_push_all_stack((ptr_t)(info.ti_stkLow),(ptr_t)(info.ti_stkHi)); return(result); } PCR_ERes GC_push_old_obj(void*p,size_t size,PCR_Any data) { GC_push_all_stack((ptr_t)p,(ptr_t)p+size); return(PCR_ERes_okay); } extern struct PCR_MM_ProcsRep*GC_old_allocator; STATIC void GC_CALLBACK GC_default_push_other_roots(void) { if ((*(GC_old_allocator->mmp_enumerate))(PCR_Bool_false, GC_push_old_obj,0) !=PCR_ERes_okay){ ABORT("Old object enumeration failed"); } if (PCR_ERes_IsErr( PCR_ThCtl_ApplyToAllOtherThreads(GC_push_thread_stack,0)) ||PCR_ERes_IsErr(GC_push_thread_stack(PCR_Th_CurrThread(),0))){ ABORT("Thread stack marking failed"); } } #elif defined(SN_TARGET_PS3) STATIC void GC_CALLBACK GC_default_push_other_roots(void) { ABORT("GC_default_push_other_roots is not implemented"); } void GC_push_thread_structures(void) { ABORT("GC_push_thread_structures is not implemented"); } #else STATIC void GC_CALLBACK GC_default_push_other_roots(void) { GC_push_all_stacks(); } #endif #endif GC_push_other_roots_proc GC_push_other_roots=GC_default_push_other_roots; GC_API void GC_CALL GC_set_push_other_roots(GC_push_other_roots_proc fn) { GC_push_other_roots=fn; } GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void) { return GC_push_other_roots; } #if (defined(CHECKSUMS)&&defined(GWW_VDB))||defined(PROC_VDB) STATIC void GC_or_pages(page_hash_table pht1,page_hash_table pht2) { unsigned i; for (i=0;i < PHT_SIZE;i++)pht1[i]|=pht2[i]; } #endif #ifdef GWW_VDB #define GC_GWW_BUF_LEN (MAXHINCR*HBLKSIZE/4096) static PVOID gww_buf[GC_GWW_BUF_LEN]; #ifndef MPROTECT_VDB #define GC_gww_dirty_init GC_dirty_init #endif GC_INNER GC_bool GC_gww_dirty_init(void) { detect_GetWriteWatch(); return GC_GWW_AVAILABLE(); } GC_INLINE void GC_gww_read_dirty(GC_bool output_unneeded) { word i; if (!output_unneeded) BZERO(GC_grungy_pages,sizeof(GC_grungy_pages)); for (i=0;i!=GC_n_heap_sects;++i){ GC_ULONG_PTR count; do { PVOID*pages=gww_buf; DWORD page_size; count=GC_GWW_BUF_LEN; if ((*(GetWriteWatch_type)(word)GetWriteWatch_func)( WRITE_WATCH_FLAG_RESET, GC_heap_sects[i].hs_start, GC_heap_sects[i].hs_bytes, pages,&count,&page_size)!=0){ static int warn_count=0; struct hblk*start=(struct hblk*)GC_heap_sects[i].hs_start; static struct hblk*last_warned=0; size_t nblocks=divHBLKSZ(GC_heap_sects[i].hs_bytes); if (i!=0&&last_warned!=start&&warn_count++< 5){ last_warned=start; WARN("GC_gww_read_dirty unexpectedly failed at %p:" "Falling back to marking all pages dirty\n",start); } if (!output_unneeded){ unsigned j; for (j=0;j < nblocks;++j){ word hash=PHT_HASH(start+j); set_pht_entry_from_index(GC_grungy_pages,hash); } } count=1; } else if (!output_unneeded){ PVOID*pages_end=pages+count; while (pages!=pages_end){ struct hblk*h=(struct hblk*)*pages++; struct hblk*h_end=(struct hblk*)((char*)h+page_size); do { set_pht_entry_from_index(GC_grungy_pages,PHT_HASH(h)); } while ((word)(++h)< (word)h_end); } } } while (count==GC_GWW_BUF_LEN); } #ifdef CHECKSUMS GC_ASSERT(!output_unneeded); GC_or_pages(GC_written_pages,GC_grungy_pages); #endif } #else #define GC_GWW_AVAILABLE()FALSE #endif #ifdef DEFAULT_VDB GC_INNER GC_bool GC_dirty_init(void) { GC_VERBOSE_LOG_PRINTF("Initializing DEFAULT_VDB...\n"); return TRUE; } #endif #ifndef GC_DISABLE_INCREMENTAL #if!defined(THREADS)||defined(HAVE_LOCKFREE_AO_OR) #define async_set_pht_entry_from_index(db,index)set_pht_entry_from_index_concurrent(db,index) #elif defined(AO_HAVE_test_and_set_acquire) GC_INNER volatile AO_TS_t GC_fault_handler_lock=AO_TS_INITIALIZER; static void async_set_pht_entry_from_index(volatile page_hash_table db, size_t index) { GC_acquire_dirty_lock(); set_pht_entry_from_index(db,index); GC_release_dirty_lock(); } #else #error No test_and_set operation:Introduces a race. #endif #endif #ifdef MPROTECT_VDB #ifdef DARWIN #include STATIC mach_port_t GC_task_self=0; #define PROTECT(addr,len)if (vm_protect(GC_task_self,(vm_address_t)(addr),(vm_size_t)(len),FALSE,VM_PROT_READ|(GC_pages_executable?VM_PROT_EXECUTE:0))==KERN_SUCCESS){} else ABORT("vm_protect(PROTECT)failed") #define UNPROTECT(addr,len)if (vm_protect(GC_task_self,(vm_address_t)(addr),(vm_size_t)(len),FALSE,(VM_PROT_READ|VM_PROT_WRITE)|(GC_pages_executable?VM_PROT_EXECUTE:0))==KERN_SUCCESS){} else ABORT("vm_protect(UNPROTECT)failed") #elif!defined(USE_WINALLOC) #include #include #if!defined(CYGWIN32)&&!defined(HAIKU) #include #endif #define PROTECT(addr,len)if (mprotect((caddr_t)(addr),(size_t)(len),PROT_READ|(GC_pages_executable?PROT_EXEC:0))>=0){ } else ABORT("mprotect failed") #define UNPROTECT(addr,len)if (mprotect((caddr_t)(addr),(size_t)(len),(PROT_READ|PROT_WRITE)|(GC_pages_executable?PROT_EXEC:0))>=0){ } else ABORT(GC_pages_executable?"un-mprotect executable page failed" " (probably disabled by OS)":"un-mprotect failed") #undef IGNORE_PAGES_EXECUTABLE #else #ifndef MSWINCE #include #endif static DWORD protect_junk; #define PROTECT(addr,len)if (VirtualProtect((addr),(len),GC_pages_executable?PAGE_EXECUTE_READ:PAGE_READONLY,&protect_junk)){ } else ABORT_ARG1("VirtualProtect failed",":errcode=0x%X",(unsigned)GetLastError()) #define UNPROTECT(addr,len)if (VirtualProtect((addr),(len),GC_pages_executable?PAGE_EXECUTE_READWRITE:PAGE_READWRITE,&protect_junk)){ } else ABORT("un-VirtualProtect failed") #endif #if defined(MSWIN32) typedef LPTOP_LEVEL_EXCEPTION_FILTER SIG_HNDLR_PTR; #undef SIG_DFL #define SIG_DFL (LPTOP_LEVEL_EXCEPTION_FILTER)((signed_word)-1) #elif defined(MSWINCE) typedef LONG (WINAPI*SIG_HNDLR_PTR)(struct _EXCEPTION_POINTERS*); #undef SIG_DFL #define SIG_DFL (SIG_HNDLR_PTR)(-1) #elif defined(DARWIN) typedef void (*SIG_HNDLR_PTR)(); #else typedef void (*SIG_HNDLR_PTR)(int,siginfo_t*,void*); typedef void (*PLAIN_HNDLR_PTR)(int); #endif #if defined(__GLIBC__) #if __GLIBC__ < 2||__GLIBC__==2&&__GLIBC_MINOR__ < 2 #error glibc too old? #endif #endif #ifndef DARWIN STATIC SIG_HNDLR_PTR GC_old_segv_handler=0; #if defined(FREEBSD)||defined(HPUX)||defined(HURD)||defined(LINUX) STATIC SIG_HNDLR_PTR GC_old_bus_handler=0; #ifndef LINUX STATIC GC_bool GC_old_bus_handler_used_si=FALSE; #endif #endif #if!defined(MSWIN32)&&!defined(MSWINCE) STATIC GC_bool GC_old_segv_handler_used_si=FALSE; #endif #endif #ifdef THREADS GC_ATTR_NO_SANITIZE_THREAD static GC_bool is_header_found_async(void*addr) { #ifdef HASH_TL hdr*result; GET_HDR((ptr_t)addr,result); return result!=NULL; #else return HDR_INNER(addr)!=NULL; #endif } #else #define is_header_found_async(addr)(HDR(addr)!=NULL) #endif #ifndef DARWIN #if!defined(MSWIN32)&&!defined(MSWINCE) #include #if defined(FREEBSD)||defined(HURD)||defined(HPUX) #define SIG_OK (sig==SIGBUS||sig==SIGSEGV) #else #define SIG_OK (sig==SIGSEGV) #endif #if defined(FREEBSD) #ifndef SEGV_ACCERR #define SEGV_ACCERR 2 #endif #if defined(AARCH64)||defined(ARM32)||defined(MIPS) #define CODE_OK (si->si_code==SEGV_ACCERR) #elif defined(POWERPC) #define AIM #include #define CODE_OK (si->si_code==EXC_DSI||si->si_code==SEGV_ACCERR) #else #define CODE_OK (si->si_code==BUS_PAGE_FAULT||si->si_code==SEGV_ACCERR) #endif #elif defined(OSF1) #define CODE_OK (si->si_code==2) #elif defined(IRIX5) #define CODE_OK (si->si_code==EACCES) #elif defined(CYGWIN32)||defined(HAIKU)||defined(HURD) #define CODE_OK TRUE #elif defined(LINUX) #define CODE_OK TRUE #elif defined(HPUX) #define CODE_OK (si->si_code==SEGV_ACCERR||si->si_code==BUS_ADRERR||si->si_code==BUS_UNKNOWN||si->si_code==SEGV_UNKNOWN||si->si_code==BUS_OBJERR) #elif defined(SUNOS5SIGS) #define CODE_OK (si->si_code==SEGV_ACCERR) #endif #ifndef NO_GETCONTEXT #include #endif STATIC void GC_write_fault_handler(int sig,siginfo_t*si,void*raw_sc) #else #define SIG_OK (exc_info->ExceptionRecord->ExceptionCode==STATUS_ACCESS_VIOLATION) #define CODE_OK (exc_info->ExceptionRecord->ExceptionInformation[0]==1) STATIC LONG WINAPI GC_write_fault_handler( struct _EXCEPTION_POINTERS*exc_info) #endif { #if!defined(MSWIN32)&&!defined(MSWINCE) char*addr=(char*)si->si_addr; #else char*addr=(char*)(exc_info->ExceptionRecord ->ExceptionInformation[1]); #endif if (SIG_OK&&CODE_OK){ struct hblk*h=(struct hblk*)((word)addr&~(GC_page_size-1)); GC_bool in_allocd_block; size_t i; GC_ASSERT(GC_page_size!=0); #ifdef CHECKSUMS GC_record_fault(h); #endif #ifdef SUNOS5SIGS in_allocd_block=FALSE; for (i=0;i < divHBLKSZ(GC_page_size);i++){ if (is_header_found_async(&h[i])){ in_allocd_block=TRUE; break; } } #else in_allocd_block=is_header_found_async(addr); #endif if (!in_allocd_block){ SIG_HNDLR_PTR old_handler; #if defined(MSWIN32)||defined(MSWINCE) old_handler=GC_old_segv_handler; #else GC_bool used_si; #if defined(FREEBSD)||defined(HURD)||defined(HPUX) if (sig==SIGBUS){ old_handler=GC_old_bus_handler; used_si=GC_old_bus_handler_used_si; } else #endif { old_handler=GC_old_segv_handler; used_si=GC_old_segv_handler_used_si; } #endif if (old_handler==(SIG_HNDLR_PTR)(signed_word)SIG_DFL){ #if!defined(MSWIN32)&&!defined(MSWINCE) ABORT_ARG1("Unexpected bus error or segmentation fault", " at %p",(void*)addr); #else return(EXCEPTION_CONTINUE_SEARCH); #endif } else { #if defined(MSWIN32)||defined(MSWINCE) return((*old_handler)(exc_info)); #else if (used_si) ((SIG_HNDLR_PTR)old_handler)(sig,si,raw_sc); else ((PLAIN_HNDLR_PTR)(signed_word)old_handler)(sig); return; #endif } } UNPROTECT(h,GC_page_size); for (i=0;i < divHBLKSZ(GC_page_size);i++){ word index=PHT_HASH(h+i); async_set_pht_entry_from_index(GC_dirty_pages,index); } #if defined(MSWIN32)||defined(MSWINCE) return(EXCEPTION_CONTINUE_EXECUTION); #else return; #endif } #if defined(MSWIN32)||defined(MSWINCE) return EXCEPTION_CONTINUE_SEARCH; #else ABORT_ARG1("Unexpected bus error or segmentation fault", " at %p",(void*)addr); #endif } #if defined(GC_WIN32_THREADS)&&!defined(CYGWIN32) GC_INNER void GC_set_write_fault_handler(void) { SetUnhandledExceptionFilter(GC_write_fault_handler); } #endif #endif #if!defined(DARWIN) GC_INNER GC_bool GC_dirty_init(void) { #if!defined(MSWIN32)&&!defined(MSWINCE) struct sigaction act,oldact; act.sa_flags=SA_RESTART|SA_SIGINFO; act.sa_sigaction=GC_write_fault_handler; (void)sigemptyset(&act.sa_mask); #if defined(THREADS)&&!defined(GC_OPENBSD_UTHREADS)&&!defined(GC_WIN32_THREADS)&&!defined(NACL) (void)sigaddset(&act.sa_mask,GC_get_suspend_signal()); #endif #endif GC_VERBOSE_LOG_PRINTF( "Initializing mprotect virtual dirty bit implementation\n"); if (GC_page_size % HBLKSIZE!=0){ ABORT("Page size not multiple of HBLKSIZE"); } #if!defined(MSWIN32)&&!defined(MSWINCE) #if defined(GC_IRIX_THREADS) sigaction(SIGSEGV,0,&oldact); sigaction(SIGSEGV,&act,0); #else { int res=sigaction(SIGSEGV,&act,&oldact); if (res!=0)ABORT("Sigaction failed"); } #endif if (oldact.sa_flags&SA_SIGINFO){ GC_old_segv_handler=oldact.sa_sigaction; GC_old_segv_handler_used_si=TRUE; } else { GC_old_segv_handler=(SIG_HNDLR_PTR)(signed_word)oldact.sa_handler; GC_old_segv_handler_used_si=FALSE; } if (GC_old_segv_handler==(SIG_HNDLR_PTR)(signed_word)SIG_IGN){ WARN("Previously ignored segmentation violation!?\n",0); GC_old_segv_handler=(SIG_HNDLR_PTR)(signed_word)SIG_DFL; } if (GC_old_segv_handler!=(SIG_HNDLR_PTR)(signed_word)SIG_DFL){ GC_VERBOSE_LOG_PRINTF("Replaced other SIGSEGV handler\n"); } #if defined(HPUX)||defined(LINUX)||defined(HURD)||(defined(FREEBSD)&&(defined(__GLIBC__)||defined(SUNOS5SIGS))) sigaction(SIGBUS,&act,&oldact); if ((oldact.sa_flags&SA_SIGINFO)!=0){ GC_old_bus_handler=oldact.sa_sigaction; #if!defined(LINUX) GC_old_bus_handler_used_si=TRUE; #endif } else { GC_old_bus_handler=(SIG_HNDLR_PTR)(signed_word)oldact.sa_handler; } if (GC_old_bus_handler==(SIG_HNDLR_PTR)(signed_word)SIG_IGN){ WARN("Previously ignored bus error!?\n",0); #if!defined(LINUX) GC_old_bus_handler=(SIG_HNDLR_PTR)(signed_word)SIG_DFL; #else #endif } else if (GC_old_bus_handler!=(SIG_HNDLR_PTR)(signed_word)SIG_DFL){ GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n"); } #endif #endif #if defined(GWW_VDB) if (GC_gww_dirty_init()) return TRUE; #endif #if defined(MSWIN32) GC_old_segv_handler=SetUnhandledExceptionFilter(GC_write_fault_handler); if (GC_old_segv_handler!=NULL){ GC_COND_LOG_PRINTF("Replaced other UnhandledExceptionFilter\n"); } else { GC_old_segv_handler=SIG_DFL; } #elif defined(MSWINCE) #endif #if defined(CPPCHECK)&&defined(ADDRESS_SANITIZER) GC_noop1((word)&__asan_default_options); #endif return TRUE; } #endif GC_API int GC_CALL GC_incremental_protection_needs(void) { GC_ASSERT(GC_is_initialized); if (GC_page_size==HBLKSIZE){ return GC_PROTECTS_POINTER_HEAP; } else { return GC_PROTECTS_POINTER_HEAP|GC_PROTECTS_PTRFREE_HEAP; } } #define HAVE_INCREMENTAL_PROTECTION_NEEDS #define IS_PTRFREE(hhdr)((hhdr)->hb_descr==0) #define PAGE_ALIGNED(x)!((word)(x)&(GC_page_size - 1)) STATIC void GC_protect_heap(void) { unsigned i; GC_bool protect_all= (0!=(GC_incremental_protection_needs()&GC_PROTECTS_PTRFREE_HEAP)); GC_ASSERT(GC_page_size!=0); for (i=0;i < GC_n_heap_sects;i++){ ptr_t start=GC_heap_sects[i].hs_start; size_t len=GC_heap_sects[i].hs_bytes; if (protect_all){ PROTECT(start,len); } else { struct hblk*current; struct hblk*current_start; struct hblk*limit; GC_ASSERT(PAGE_ALIGNED(len)); GC_ASSERT(PAGE_ALIGNED(start)); current_start=current=(struct hblk*)start; limit=(struct hblk*)(start+len); while ((word)current < (word)limit){ hdr*hhdr; word nhblks; GC_bool is_ptrfree; GC_ASSERT(PAGE_ALIGNED(current)); GET_HDR(current,hhdr); if (IS_FORWARDING_ADDR_OR_NIL(hhdr)){ GC_ASSERT(current_start==current); current_start=++current; continue; } if (HBLK_IS_FREE(hhdr)){ GC_ASSERT(PAGE_ALIGNED(hhdr->hb_sz)); nhblks=divHBLKSZ(hhdr->hb_sz); is_ptrfree=TRUE; } else { nhblks=OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); is_ptrfree=IS_PTRFREE(hhdr); } if (is_ptrfree){ if ((word)current_start < (word)current){ PROTECT(current_start,(ptr_t)current - (ptr_t)current_start); } current_start=(current+=nhblks); } else { current+=nhblks; } } if ((word)current_start < (word)current){ PROTECT(current_start,(ptr_t)current - (ptr_t)current_start); } } } } #endif #ifdef PROC_VDB #include #include #include #include #include #ifdef GC_NO_SYS_FAULT_H #define PG_MODIFIED 1 struct prpageheader { int dummy[2]; unsigned long pr_nmap; unsigned long pr_npage; }; struct prasmap { char*pr_vaddr; size_t pr_npage; char dummy1[64+8]; unsigned pr_mflags; unsigned pr_pagesize; int dummy2[2]; }; #else #include #include #endif #define INITIAL_BUF_SZ 16384 STATIC size_t GC_proc_buf_size=INITIAL_BUF_SZ; STATIC char*GC_proc_buf=NULL; STATIC int GC_proc_fd=0; GC_INNER GC_bool GC_dirty_init(void) { char buf[40]; if (GC_bytes_allocd!=0||GC_bytes_allocd_before_gc!=0){ memset(GC_written_pages,0xff,sizeof(page_hash_table)); GC_VERBOSE_LOG_PRINTF( "Allocated %lu bytes:all pages may have been written\n", (unsigned long)(GC_bytes_allocd+GC_bytes_allocd_before_gc)); } (void)snprintf(buf,sizeof(buf),"/proc/%ld/pagedata",(long)getpid()); buf[sizeof(buf)- 1]='\0'; GC_proc_fd=open(buf,O_RDONLY); if (GC_proc_fd < 0){ WARN("/proc open failed;cannot enable GC incremental mode\n",0); return FALSE; } if (syscall(SYS_fcntl,GC_proc_fd,F_SETFD,FD_CLOEXEC)==-1) WARN("Could not set FD_CLOEXEC for/proc\n",0); GC_proc_buf=GC_scratch_alloc(GC_proc_buf_size); if (GC_proc_buf==NULL) ABORT("Insufficient space for/proc read"); return TRUE; } GC_INLINE void GC_proc_read_dirty(GC_bool output_unneeded) { #define READ read int nmaps; char*bufp=GC_proc_buf; int i; BZERO(GC_grungy_pages,sizeof(GC_grungy_pages)); if (READ(GC_proc_fd,bufp,GC_proc_buf_size)<=0){ size_t new_size=2*GC_proc_buf_size; char*new_buf; WARN("/proc read failed:GC_proc_buf_size=%" WARN_PRIdPTR "\n", (signed_word)GC_proc_buf_size); new_buf=GC_scratch_alloc(new_size); if (new_buf!=0){ GC_scratch_recycle_no_gww(bufp,GC_proc_buf_size); GC_proc_buf=bufp=new_buf; GC_proc_buf_size=new_size; } if (READ(GC_proc_fd,bufp,GC_proc_buf_size)<=0){ WARN("Insufficient space for/proc read\n",0); if (!output_unneeded) memset(GC_grungy_pages,0xff,sizeof (page_hash_table)); memset(GC_written_pages,0xff,sizeof(page_hash_table)); return; } } nmaps=((struct prpageheader*)bufp)->pr_nmap; #ifdef DEBUG_DIRTY_BITS GC_log_printf("Proc VDB read:pr_nmap=%u,pr_npage=%lu\n", nmaps,((struct prpageheader*)bufp)->pr_npage); #endif #if defined(GC_NO_SYS_FAULT_H)&&defined(CPPCHECK) GC_noop1(((struct prpageheader*)bufp)->dummy[0]); #endif bufp+=sizeof(struct prpageheader); for (i=0;i < nmaps;i++){ struct prasmap*map=(struct prasmap*)bufp; ptr_t vaddr=(ptr_t)(map->pr_vaddr); unsigned long npages=map->pr_npage; unsigned pagesize=map->pr_pagesize; ptr_t limit; #if defined(GC_NO_SYS_FAULT_H)&&defined(CPPCHECK) GC_noop1(map->dummy1[0]+map->dummy2[0]); #endif #ifdef DEBUG_DIRTY_BITS GC_log_printf( "pr_vaddr=%p,npage=%lu,mflags=0x%x,pagesize=0x%x\n", (void*)vaddr,npages,map->pr_mflags,pagesize); #endif bufp+=sizeof(struct prasmap); limit=vaddr+pagesize*npages; for (;(word)vaddr < (word)limit;vaddr+=pagesize){ if ((*bufp++)&PG_MODIFIED){ struct hblk*h; ptr_t next_vaddr=vaddr+pagesize; #ifdef DEBUG_DIRTY_BITS GC_log_printf("dirty page at:%p\n",(void*)vaddr); #endif for (h=(struct hblk*)vaddr; (word)h < (word)next_vaddr;h++){ word index=PHT_HASH(h); set_pht_entry_from_index(GC_grungy_pages,index); } } } bufp=(char*)(((word)bufp+(sizeof(long)-1)) &~(word)(sizeof(long)-1)); } #ifdef DEBUG_DIRTY_BITS GC_log_printf("Proc VDB read done\n"); #endif GC_or_pages(GC_written_pages,GC_grungy_pages); #undef READ } #endif #ifdef PCR_VDB #include "vd/PCR_VD.h" #define NPAGES (32*1024) PCR_VD_DB GC_grungy_bits[NPAGES]; STATIC ptr_t GC_vd_base=NULL; GC_INNER GC_bool GC_dirty_init(void) { GC_vd_base=GC_heap_sects[0].hs_start; if (GC_vd_base==0){ ABORT("Bad initial heap segment"); } if (PCR_VD_Start(HBLKSIZE,GC_vd_base,NPAGES*HBLKSIZE) !=PCR_ERes_okay){ ABORT("Dirty bit initialization failed"); } return TRUE; } #endif #ifndef GC_DISABLE_INCREMENTAL GC_INNER GC_bool GC_manual_vdb=FALSE; GC_INNER void GC_dirty_inner(const void*p) { word index=PHT_HASH(p); #if defined(MPROTECT_VDB) GC_ASSERT(GC_manual_vdb); #endif async_set_pht_entry_from_index(GC_dirty_pages,index); } GC_INNER void GC_read_dirty(GC_bool output_unneeded) { if (GC_manual_vdb #if defined(MPROTECT_VDB) ||!GC_GWW_AVAILABLE() #endif ){ if (!output_unneeded) BCOPY(( void*)GC_dirty_pages,GC_grungy_pages, sizeof(GC_dirty_pages)); BZERO(( void*)GC_dirty_pages, sizeof(GC_dirty_pages)); #ifdef MPROTECT_VDB if (!GC_manual_vdb) GC_protect_heap(); #endif return; } #ifdef GWW_VDB GC_gww_read_dirty(output_unneeded); #elif defined(PROC_VDB) GC_proc_read_dirty(output_unneeded); #elif defined(PCR_VDB) { static int onhs=0; int nhs=GC_n_heap_sects; for (;onhs < nhs;onhs++){ PCR_VD_WriteProtectEnable( GC_heap_sects[onhs].hs_start, GC_heap_sects[onhs].hs_bytes); } } if (PCR_VD_Clear(GC_vd_base,NPAGES*HBLKSIZE,GC_grungy_bits) !=PCR_ERes_okay){ ABORT("Dirty bit read failed"); } #endif } GC_INNER GC_bool GC_page_was_dirty(struct hblk*h) { #ifdef PCR_VDB if (!GC_manual_vdb){ if ((word)h < (word)GC_vd_base ||(word)h>=(word)(GC_vd_base+NPAGES*HBLKSIZE)){ return TRUE; } return GC_grungy_bits[h-(struct hblk*)GC_vd_base]&PCR_VD_DB_dirtyBit; } #elif defined(DEFAULT_VDB) if (!GC_manual_vdb) return TRUE; #endif return NULL==HDR(h) ||get_pht_entry_from_index(GC_grungy_pages,PHT_HASH(h)); } #if defined(CHECKSUMS)||defined(PROC_VDB) GC_INNER GC_bool GC_page_was_ever_dirty(struct hblk*h) { #if defined(GWW_VDB)||defined(PROC_VDB) #ifdef MPROTECT_VDB if (!GC_GWW_AVAILABLE()) return TRUE; #endif return NULL==HDR(h) ||get_pht_entry_from_index(GC_written_pages,PHT_HASH(h)); #else (void)h; return TRUE; #endif } #endif GC_INNER void GC_remove_protection(struct hblk*h,word nblocks, GC_bool is_ptrfree) { #ifdef PCR_VDB (void)is_ptrfree; if (!GC_auto_incremental) return; PCR_VD_WriteProtectDisable(h,nblocks*HBLKSIZE); PCR_VD_WriteProtectEnable(h,nblocks*HBLKSIZE); #elif defined(MPROTECT_VDB) struct hblk*h_trunc; struct hblk*h_end; struct hblk*current; if (!GC_auto_incremental||GC_GWW_AVAILABLE()) return; GC_ASSERT(GC_page_size!=0); h_trunc=(struct hblk*)((word)h&~(GC_page_size-1)); h_end=(struct hblk*)(((word)(h+nblocks)+GC_page_size - 1) &~(GC_page_size - 1)); if (h_end==h_trunc+1&& get_pht_entry_from_index(GC_dirty_pages,PHT_HASH(h_trunc))){ return; } for (current=h_trunc;(word)current < (word)h_end;++current){ word index=PHT_HASH(current); if (!is_ptrfree||(word)current < (word)h ||(word)current>=(word)(h+nblocks)){ async_set_pht_entry_from_index(GC_dirty_pages,index); } } UNPROTECT(h_trunc,(ptr_t)h_end - (ptr_t)h_trunc); #else (void)h;(void)nblocks;(void)is_ptrfree; #endif } #endif #if defined(MPROTECT_VDB)&&defined(DARWIN) #include #include #include #include #include EXTERN_C_BEGIN extern boolean_t exc_server(mach_msg_header_t*,mach_msg_header_t*); extern kern_return_t exception_raise(mach_port_t,mach_port_t,mach_port_t,exception_type_t, exception_data_t,mach_msg_type_number_t); extern kern_return_t exception_raise_state(mach_port_t,mach_port_t,mach_port_t,exception_type_t, exception_data_t,mach_msg_type_number_t, thread_state_flavor_t*,thread_state_t, mach_msg_type_number_t,thread_state_t, mach_msg_type_number_t*); extern kern_return_t exception_raise_state_identity(mach_port_t,mach_port_t,mach_port_t, exception_type_t,exception_data_t, mach_msg_type_number_t,thread_state_flavor_t*, thread_state_t,mach_msg_type_number_t, thread_state_t,mach_msg_type_number_t*); GC_API_OSCALL kern_return_t catch_exception_raise(mach_port_t exception_port,mach_port_t thread, mach_port_t task,exception_type_t exception, exception_data_t code, mach_msg_type_number_t code_count); GC_API_OSCALL kern_return_t catch_exception_raise_state(mach_port_name_t exception_port, int exception,exception_data_t code, mach_msg_type_number_t codeCnt,int flavor, thread_state_t old_state,int old_stateCnt, thread_state_t new_state,int new_stateCnt); GC_API_OSCALL kern_return_t catch_exception_raise_state_identity(mach_port_name_t exception_port, mach_port_t thread,mach_port_t task,int exception, exception_data_t code,mach_msg_type_number_t codeCnt, int flavor,thread_state_t old_state,int old_stateCnt, thread_state_t new_state,int new_stateCnt); EXTERN_C_END GC_API_OSCALL kern_return_t catch_exception_raise_state(mach_port_name_t exception_port GC_ATTR_UNUSED, int exception GC_ATTR_UNUSED,exception_data_t code GC_ATTR_UNUSED, mach_msg_type_number_t codeCnt GC_ATTR_UNUSED,int flavor GC_ATTR_UNUSED, thread_state_t old_state GC_ATTR_UNUSED,int old_stateCnt GC_ATTR_UNUSED, thread_state_t new_state GC_ATTR_UNUSED,int new_stateCnt GC_ATTR_UNUSED) { ABORT_RET("Unexpected catch_exception_raise_state invocation"); return(KERN_INVALID_ARGUMENT); } GC_API_OSCALL kern_return_t catch_exception_raise_state_identity( mach_port_name_t exception_port GC_ATTR_UNUSED, mach_port_t thread GC_ATTR_UNUSED,mach_port_t task GC_ATTR_UNUSED, int exception GC_ATTR_UNUSED,exception_data_t code GC_ATTR_UNUSED, mach_msg_type_number_t codeCnt GC_ATTR_UNUSED,int flavor GC_ATTR_UNUSED, thread_state_t old_state GC_ATTR_UNUSED,int old_stateCnt GC_ATTR_UNUSED, thread_state_t new_state GC_ATTR_UNUSED,int new_stateCnt GC_ATTR_UNUSED) { ABORT_RET("Unexpected catch_exception_raise_state_identity invocation"); return(KERN_INVALID_ARGUMENT); } #define MAX_EXCEPTION_PORTS 16 static struct { mach_msg_type_number_t count; exception_mask_t masks[MAX_EXCEPTION_PORTS]; exception_handler_t ports[MAX_EXCEPTION_PORTS]; exception_behavior_t behaviors[MAX_EXCEPTION_PORTS]; thread_state_flavor_t flavors[MAX_EXCEPTION_PORTS]; } GC_old_exc_ports; STATIC struct ports_s { void (*volatile os_callback[3])(void); mach_port_t exception; #if defined(THREADS) mach_port_t reply; #endif } GC_ports={ { (void (*)(void))catch_exception_raise, (void (*)(void))catch_exception_raise_state, (void (*)(void))catch_exception_raise_state_identity }, #ifdef THREADS 0, #endif 0 }; typedef struct { mach_msg_header_t head; } GC_msg_t; typedef enum { GC_MP_NORMAL, GC_MP_DISCARDING, GC_MP_STOPPED } GC_mprotect_state_t; #ifdef THREADS #define ID_STOP 1 #define ID_RESUME 2 #define ID_ACK 3 STATIC GC_mprotect_state_t GC_mprotect_state=GC_MP_NORMAL; STATIC void GC_mprotect_thread_notify(mach_msg_id_t id) { struct buf_s { GC_msg_t msg; mach_msg_trailer_t trailer; } buf; mach_msg_return_t r; buf.msg.head.msgh_bits=MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND,0); buf.msg.head.msgh_size=sizeof(buf.msg); buf.msg.head.msgh_remote_port=GC_ports.exception; buf.msg.head.msgh_local_port=MACH_PORT_NULL; buf.msg.head.msgh_id=id; r=mach_msg(&buf.msg.head,MACH_SEND_MSG|MACH_RCV_MSG|MACH_RCV_LARGE, sizeof(buf.msg),sizeof(buf),GC_ports.reply, MACH_MSG_TIMEOUT_NONE,MACH_PORT_NULL); if (r!=MACH_MSG_SUCCESS) ABORT("mach_msg failed in GC_mprotect_thread_notify"); if (buf.msg.head.msgh_id!=ID_ACK) ABORT("Invalid ack in GC_mprotect_thread_notify"); } STATIC void GC_mprotect_thread_reply(void) { GC_msg_t msg; mach_msg_return_t r; msg.head.msgh_bits=MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND,0); msg.head.msgh_size=sizeof(msg); msg.head.msgh_remote_port=GC_ports.reply; msg.head.msgh_local_port=MACH_PORT_NULL; msg.head.msgh_id=ID_ACK; r=mach_msg(&msg.head,MACH_SEND_MSG,sizeof(msg),0,MACH_PORT_NULL, MACH_MSG_TIMEOUT_NONE,MACH_PORT_NULL); if (r!=MACH_MSG_SUCCESS) ABORT("mach_msg failed in GC_mprotect_thread_reply"); } GC_INNER void GC_mprotect_stop(void) { GC_mprotect_thread_notify(ID_STOP); } GC_INNER void GC_mprotect_resume(void) { GC_mprotect_thread_notify(ID_RESUME); } #else #define GC_mprotect_state GC_MP_NORMAL #endif struct mp_reply_s { mach_msg_header_t head; char data[256]; }; struct mp_msg_s { mach_msg_header_t head; mach_msg_body_t msgh_body; char data[1024]; }; STATIC void*GC_mprotect_thread(void*arg) { mach_msg_return_t r; struct mp_reply_s reply; struct mp_msg_s msg; mach_msg_id_t id; if ((word)arg==GC_WORD_MAX)return 0; #if defined(CPPCHECK) reply.data[0]=0; msg.data[0]=0; #endif #if defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID) (void)pthread_setname_np("GC-mprotect"); #endif #if defined(THREADS)&&!defined(GC_NO_THREADS_DISCOVERY) GC_darwin_register_mach_handler_thread(mach_thread_self()); #endif for(;;){ r=mach_msg(&msg.head,MACH_RCV_MSG|MACH_RCV_LARGE| (GC_mprotect_state==GC_MP_DISCARDING?MACH_RCV_TIMEOUT:0), 0,sizeof(msg),GC_ports.exception, GC_mprotect_state==GC_MP_DISCARDING?0 :MACH_MSG_TIMEOUT_NONE,MACH_PORT_NULL); id=r==MACH_MSG_SUCCESS?msg.head.msgh_id:-1; #if defined(THREADS) if(GC_mprotect_state==GC_MP_DISCARDING){ if(r==MACH_RCV_TIMED_OUT){ GC_mprotect_state=GC_MP_STOPPED; GC_mprotect_thread_reply(); continue; } if(r==MACH_MSG_SUCCESS&&(id==ID_STOP||id==ID_RESUME)) ABORT("Out of order mprotect thread request"); } #endif if (r!=MACH_MSG_SUCCESS){ ABORT_ARG2("mach_msg failed", ":errcode=%d (%s)",(int)r,mach_error_string(r)); } switch(id){ #if defined(THREADS) case ID_STOP: if(GC_mprotect_state!=GC_MP_NORMAL) ABORT("Called mprotect_stop when state wasn't normal"); GC_mprotect_state=GC_MP_DISCARDING; break; case ID_RESUME: if(GC_mprotect_state!=GC_MP_STOPPED) ABORT("Called mprotect_resume when state wasn't stopped"); GC_mprotect_state=GC_MP_NORMAL; GC_mprotect_thread_reply(); break; #endif default: if(!exc_server(&msg.head,&reply.head)) ABORT("exc_server failed"); r=mach_msg(&reply.head,MACH_SEND_MSG,reply.head.msgh_size,0, MACH_PORT_NULL,MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL); if(r!=MACH_MSG_SUCCESS){ #ifdef BROKEN_EXCEPTION_HANDLING GC_err_printf("mach_msg failed with %d %s while sending " "exc reply\n",(int)r,mach_error_string(r)); #else ABORT("mach_msg failed while sending exception reply"); #endif } } } } #ifdef BROKEN_EXCEPTION_HANDLING STATIC int GC_sigbus_count=0; STATIC void GC_darwin_sigbus(int num,siginfo_t*sip,void*context) { if (num!=SIGBUS) ABORT("Got a non-sigbus signal in the sigbus handler"); if (GC_sigbus_count>=8){ ABORT("Got more than 8 SIGBUSs in a row!"); } else { GC_sigbus_count++; WARN("Ignoring SIGBUS\n",0); } } #endif GC_INNER GC_bool GC_dirty_init(void) { kern_return_t r; mach_port_t me; pthread_t thread; pthread_attr_t attr; exception_mask_t mask; #ifdef CAN_HANDLE_FORK if (GC_handle_fork){ WARN("Can't turn on GC incremental mode as fork()" " handling requested\n",0); return FALSE; } #endif GC_VERBOSE_LOG_PRINTF("Initializing mach/darwin mprotect" " virtual dirty bit implementation\n"); #ifdef BROKEN_EXCEPTION_HANDLING WARN("Enabling workarounds for various darwin " "exception handling bugs\n",0); #endif if (GC_page_size % HBLKSIZE!=0){ ABORT("Page size not multiple of HBLKSIZE"); } GC_task_self=me=mach_task_self(); r=mach_port_allocate(me,MACH_PORT_RIGHT_RECEIVE,&GC_ports.exception); if (r!=KERN_SUCCESS) ABORT("mach_port_allocate failed (exception port)"); r=mach_port_insert_right(me,GC_ports.exception,GC_ports.exception, MACH_MSG_TYPE_MAKE_SEND); if (r!=KERN_SUCCESS) ABORT("mach_port_insert_right failed (exception port)"); #if defined(THREADS) r=mach_port_allocate(me,MACH_PORT_RIGHT_RECEIVE,&GC_ports.reply); if(r!=KERN_SUCCESS) ABORT("mach_port_allocate failed (reply port)"); #endif mask=EXC_MASK_BAD_ACCESS; r=task_get_exception_ports(me,mask,GC_old_exc_ports.masks, &GC_old_exc_ports.count,GC_old_exc_ports.ports, GC_old_exc_ports.behaviors, GC_old_exc_ports.flavors); if (r!=KERN_SUCCESS) ABORT("task_get_exception_ports failed"); r=task_set_exception_ports(me,mask,GC_ports.exception,EXCEPTION_DEFAULT, GC_MACH_THREAD_STATE); if (r!=KERN_SUCCESS) ABORT("task_set_exception_ports failed"); if (pthread_attr_init(&attr)!=0) ABORT("pthread_attr_init failed"); if (pthread_attr_setdetachstate(&attr,PTHREAD_CREATE_DETACHED)!=0) ABORT("pthread_attr_setdetachedstate failed"); #undef pthread_create if (pthread_create(&thread,&attr,GC_mprotect_thread,NULL)!=0) ABORT("pthread_create failed"); (void)pthread_attr_destroy(&attr); #ifdef BROKEN_EXCEPTION_HANDLING { struct sigaction sa,oldsa; sa.sa_handler=(SIG_HNDLR_PTR)GC_darwin_sigbus; sigemptyset(&sa.sa_mask); sa.sa_flags=SA_RESTART|SA_SIGINFO; if (sigaction(SIGBUS,&sa,&oldsa)< 0) ABORT("sigaction failed"); if (oldsa.sa_handler!=(SIG_HNDLR_PTR)(signed_word)SIG_DFL){ GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n"); } } #endif #if defined(CPPCHECK) GC_noop1((word)GC_ports.os_callback[0]); #endif return TRUE; } STATIC kern_return_t GC_forward_exception(mach_port_t thread,mach_port_t task, exception_type_t exception, exception_data_t data, mach_msg_type_number_t data_count) { unsigned int i; kern_return_t r; mach_port_t port; exception_behavior_t behavior; thread_state_flavor_t flavor; thread_state_data_t thread_state; mach_msg_type_number_t thread_state_count=THREAD_STATE_MAX; for (i=0;i < GC_old_exc_ports.count;i++) if (GC_old_exc_ports.masks[i]&(1< 0?code[0]:-1, code_count > 1?code[1]:-1); #endif return FWD(); } r=thread_get_state(thread,flavor,(natural_t*)&exc_state, &exc_state_count); if(r!=KERN_SUCCESS){ #ifdef BROKEN_EXCEPTION_HANDLING GC_err_printf("thread_get_state failed in catch_exception_raise\n"); return KERN_SUCCESS; #else ABORT("thread_get_state failed in catch_exception_raise"); #endif } addr=(char*)exc_state.DARWIN_EXC_STATE_DAR; if (!is_header_found_async(addr)){ #ifdef BROKEN_EXCEPTION_HANDLING static char*last_fault; static int last_fault_count; if(addr!=last_fault){ last_fault=addr; last_fault_count=0; } if(++last_fault_count < 32){ if(last_fault_count==1) WARN("Ignoring KERN_PROTECTION_FAILURE at %p\n",addr); return KERN_SUCCESS; } GC_err_printf("Unexpected KERN_PROTECTION_FAILURE at %p;aborting...\n", (void*)addr); EXIT(); #else return FWD(); #endif } #ifdef BROKEN_EXCEPTION_HANDLING GC_sigbus_count=0; #endif GC_ASSERT(GC_page_size!=0); if (GC_mprotect_state==GC_MP_NORMAL){ struct hblk*h=(struct hblk*)((word)addr&~(GC_page_size-1)); size_t i; UNPROTECT(h,GC_page_size); for (i=0;i < divHBLKSZ(GC_page_size);i++){ word index=PHT_HASH(h+i); async_set_pht_entry_from_index(GC_dirty_pages,index); } } else if (GC_mprotect_state==GC_MP_DISCARDING){ } else { GC_err_printf("KERN_PROTECTION_FAILURE while world is stopped\n"); return FWD(); } return KERN_SUCCESS; } #undef FWD #ifndef NO_DESC_CATCH_EXCEPTION_RAISE __asm__(".desc _catch_exception_raise,0x10"); __asm__(".desc _catch_exception_raise_state,0x10"); __asm__(".desc _catch_exception_raise_state_identity,0x10"); #endif #endif #ifndef HAVE_INCREMENTAL_PROTECTION_NEEDS GC_API int GC_CALL GC_incremental_protection_needs(void) { GC_ASSERT(GC_is_initialized); return GC_PROTECTS_NONE; } #endif #ifdef ECOS #undef sbrk #endif GC_API void GC_CALL GC_set_pages_executable(int value) { GC_ASSERT(!GC_is_initialized); GC_pages_executable=(GC_bool)(value!=0); } GC_API int GC_CALL GC_get_pages_executable(void) { #ifdef IGNORE_PAGES_EXECUTABLE return 1; #else return (int)GC_pages_executable; #endif } #if defined(I386)&&defined(LINUX)&&defined(SAVE_CALL_CHAIN) #include struct frame { struct frame*fr_savfp; long fr_savpc; #if NARGS > 0 long fr_arg[NARGS]; #endif }; #endif #if defined(SPARC) #if defined(LINUX) #include #if defined(SAVE_CALL_CHAIN) struct frame { long fr_local[8]; long fr_arg[6]; struct frame*fr_savfp; long fr_savpc; #ifndef __arch64__ char*fr_stret; #endif long fr_argd[6]; long fr_argx[0]; }; #endif #elif defined (DRSNX) #include #elif defined(OPENBSD) #include #elif defined(FREEBSD)||defined(NETBSD) #include #else #include #endif #if NARGS > 6 #error We only know how to get the first 6 arguments #endif #endif #ifdef NEED_CALLINFO #ifdef LINUX #include #endif #endif #if defined(GC_HAVE_BUILTIN_BACKTRACE) #ifdef _MSC_VER #ifndef GC_MSVC_DBG_H #define GC_MSVC_DBG_H #include #ifdef __cplusplus extern "C" { #endif #if!MSVC_DBG_DLL #define MSVC_DBG_EXPORT #elif MSVC_DBG_BUILD #define MSVC_DBG_EXPORT __declspec(dllexport) #else #define MSVC_DBG_EXPORT __declspec(dllimport) #endif #ifndef MAX_SYM_NAME #define MAX_SYM_NAME 2000 #endif typedef void*HANDLE; typedef struct _CONTEXT CONTEXT; MSVC_DBG_EXPORT size_t GetStackFrames(size_t skip,void*frames[],size_t maxFrames); MSVC_DBG_EXPORT size_t GetStackFramesFromContext(HANDLE hProcess,HANDLE hThread,CONTEXT*context,size_t skip,void*frames[],size_t maxFrames); MSVC_DBG_EXPORT size_t GetModuleNameFromAddress(void*address,char*moduleName,size_t size); MSVC_DBG_EXPORT size_t GetModuleNameFromStack(size_t skip,char*moduleName,size_t size); MSVC_DBG_EXPORT size_t GetSymbolNameFromAddress(void*address,char*symbolName,size_t size,size_t*offsetBytes); MSVC_DBG_EXPORT size_t GetSymbolNameFromStack(size_t skip,char*symbolName,size_t size,size_t*offsetBytes); MSVC_DBG_EXPORT size_t GetFileLineFromAddress(void*address,char*fileName,size_t size,size_t*lineNumber,size_t*offsetBytes); MSVC_DBG_EXPORT size_t GetFileLineFromStack(size_t skip,char*fileName,size_t size,size_t*lineNumber,size_t*offsetBytes); MSVC_DBG_EXPORT size_t GetDescriptionFromAddress(void*address,const char*format,char*description,size_t size); MSVC_DBG_EXPORT size_t GetDescriptionFromStack(void*const frames[],size_t count,const char*format,char*description[],size_t size); MSVC_DBG_EXPORT int backtrace(void*addresses[],int count); MSVC_DBG_EXPORT char**backtrace_symbols(void*const addresses[],int count); #ifdef __cplusplus } #endif #endif #else #include #endif #endif #ifdef SAVE_CALL_CHAIN #if NARGS==0&&NFRAMES % 2==0&&defined(GC_HAVE_BUILTIN_BACKTRACE) #ifdef REDIRECT_MALLOC #ifdef THREADS __thread #endif GC_bool GC_in_save_callers=FALSE; #endif GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]) { void*tmp_info[NFRAMES+1]; int npcs,i; #define IGNORE_FRAMES 1 #ifdef REDIRECT_MALLOC if (GC_in_save_callers){ info[0].ci_pc=(word)(&GC_save_callers); for (i=1;i < NFRAMES;++i)info[i].ci_pc=0; return; } GC_in_save_callers=TRUE; #endif GC_ASSERT(I_HOLD_LOCK()); GC_STATIC_ASSERT(sizeof(struct callinfo)==sizeof(void*)); npcs=backtrace((void**)tmp_info,NFRAMES+IGNORE_FRAMES); if (npcs > IGNORE_FRAMES) BCOPY(&tmp_info[IGNORE_FRAMES],info, (npcs - IGNORE_FRAMES)*sizeof(void*)); for (i=npcs - IGNORE_FRAMES;i < NFRAMES;++i)info[i].ci_pc=0; #ifdef REDIRECT_MALLOC GC_in_save_callers=FALSE; #endif } #else #if (defined(OPENBSD)||defined(NETBSD)||defined(FREEBSD))&&defined(SPARC) #define FR_SAVFP fr_fp #define FR_SAVPC fr_pc #else #define FR_SAVFP fr_savfp #define FR_SAVPC fr_savpc #endif #if defined(SPARC)&&(defined(__arch64__)||defined(__sparcv9)) #define BIAS 2047 #else #define BIAS 0 #endif GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]) { struct frame*frame; struct frame*fp; int nframes=0; #ifdef I386 asm("movl %%ebp,%0":"=r"(frame)); fp=frame; #else frame=(struct frame*)GC_save_regs_in_stack(); fp=(struct frame*)((long)frame->FR_SAVFP+BIAS); #endif for (;!((word)fp HOTTER_THAN (word)frame) #ifndef THREADS &&!((word)GC_stackbottom HOTTER_THAN (word)fp) #elif defined(STACK_GROWS_UP) &&fp!=NULL #endif &&nframes < NFRAMES; fp=(struct frame*)((long)fp->FR_SAVFP+BIAS),nframes++){ #if NARGS > 0 int i; #endif info[nframes].ci_pc=fp->FR_SAVPC; #if NARGS > 0 for (i=0;i < NARGS;i++){ info[nframes].ci_arg[i]=~(fp->fr_arg[i]); } #endif } if (nframes < NFRAMES)info[nframes].ci_pc=0; } #endif #endif #ifdef NEED_CALLINFO GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]) { int i; static int reentry_count=0; DCL_LOCK_STATE; LOCK(); ++reentry_count; UNLOCK(); #if NFRAMES==1 GC_err_printf("\tCaller at allocation:\n"); #else GC_err_printf("\tCall chain at allocation:\n"); #endif for (i=0;i < NFRAMES;i++){ #if defined(LINUX)&&!defined(SMALL_CONFIG) GC_bool stop=FALSE; #endif if (0==info[i].ci_pc) break; #if NARGS > 0 { int j; GC_err_printf("\t\targs:"); for (j=0;j < NARGS;j++){ if (j!=0)GC_err_printf(","); GC_err_printf("%d (0x%X)",~(info[i].ci_arg[j]), ~(info[i].ci_arg[j])); } GC_err_printf("\n"); } #endif if (reentry_count > 1){ GC_err_printf("\t\t##PC##=0x%lx\n", (unsigned long)info[i].ci_pc); continue; } { char buf[40]; char*name; #if defined(GC_HAVE_BUILTIN_BACKTRACE)&&!defined(GC_BACKTRACE_SYMBOLS_BROKEN) char**sym_name= backtrace_symbols((void**)(&(info[i].ci_pc)),1); if (sym_name!=NULL){ name=sym_name[0]; } else #endif { (void)snprintf(buf,sizeof(buf),"##PC##=0x%lx", (unsigned long)info[i].ci_pc); buf[sizeof(buf)- 1]='\0'; name=buf; } #if defined(LINUX)&&!defined(SMALL_CONFIG) do { FILE*pipe; #define EXE_SZ 100 static char exe_name[EXE_SZ]; #define CMD_SZ 200 char cmd_buf[CMD_SZ]; #define RESULT_SZ 200 static char result_buf[RESULT_SZ]; size_t result_len; char*old_preload; #define PRELOAD_SZ 200 char preload_buf[PRELOAD_SZ]; static GC_bool found_exe_name=FALSE; static GC_bool will_fail=FALSE; if (will_fail) break; if (!found_exe_name){ int ret_code=readlink("/proc/self/exe",exe_name,EXE_SZ); if (ret_code < 0||ret_code>=EXE_SZ ||exe_name[0]!='/'){ will_fail=TRUE; break; } exe_name[ret_code]='\0'; found_exe_name=TRUE; } (void)snprintf(cmd_buf,sizeof(cmd_buf), "/usr/bin/addr2line -f -e %s 0x%lx", exe_name,(unsigned long)info[i].ci_pc); cmd_buf[sizeof(cmd_buf)- 1]='\0'; old_preload=GETENV("LD_PRELOAD"); if (0!=old_preload){ size_t old_len=strlen(old_preload); if (old_len>=PRELOAD_SZ){ will_fail=TRUE; break; } BCOPY(old_preload,preload_buf,old_len+1); unsetenv ("LD_PRELOAD"); } pipe=popen(cmd_buf,"r"); if (0!=old_preload &&0!=setenv ("LD_PRELOAD",preload_buf,0)){ WARN("Failed to reset LD_PRELOAD\n",0); } if (NULL==pipe){ will_fail=TRUE; break; } result_len=fread(result_buf,1,RESULT_SZ - 1,pipe); (void)pclose(pipe); if (0==result_len){ will_fail=TRUE; break; } if (result_buf[result_len - 1]=='\n')--result_len; result_buf[result_len]=0; if (result_buf[0]=='?' ||(result_buf[result_len-2]==':' &&result_buf[result_len-1]=='0')) break; { char*nl=strchr(result_buf,'\n'); if (nl!=NULL &&(word)nl < (word)(result_buf+result_len)){ *nl=':'; } if (strncmp(result_buf,"main", nl!=NULL ?(size_t)((word)nl - COVERT_DATAFLOW(result_buf)) :result_len)==0){ stop=TRUE; } } if (result_len < RESULT_SZ - 25){ (void)snprintf(&result_buf[result_len], sizeof(result_buf)- result_len, " [0x%lx]",(unsigned long)info[i].ci_pc); result_buf[sizeof(result_buf)- 1]='\0'; } #if defined(CPPCHECK) GC_noop1((unsigned char)name[0]); #endif name=result_buf; } while (0); #endif GC_err_printf("\t\t%s\n",name); #if defined(GC_HAVE_BUILTIN_BACKTRACE)&&!defined(GC_BACKTRACE_SYMBOLS_BROKEN) if (sym_name!=NULL) free(sym_name); #endif } #if defined(LINUX)&&!defined(SMALL_CONFIG) if (stop) break; #endif } LOCK(); --reentry_count; UNLOCK(); } #endif #if defined(LINUX)&&defined(__ELF__)&&!defined(SMALL_CONFIG) void GC_print_address_map(void) { char*maps; GC_err_printf("----------Begin address map----------\n"); maps=GC_get_maps(); GC_err_puts(maps!=NULL?maps:"Failed to get map!\n"); GC_err_printf("----------End address map----------\n"); } #endif #if defined(THREAD_LOCAL_ALLOC) #ifndef THREADS #error "invalid config - THREAD_LOCAL_ALLOC requires GC_THREADS" #endif #ifndef GC_THREAD_LOCAL_ALLOC_H #define GC_THREAD_LOCAL_ALLOC_H #ifdef THREAD_LOCAL_ALLOC #if defined(USE_HPUX_TLS) #error USE_HPUX_TLS macro was replaced by USE_COMPILER_TLS #endif #include EXTERN_C_BEGIN #if!defined(USE_PTHREAD_SPECIFIC)&&!defined(USE_WIN32_SPECIFIC)&&!defined(USE_WIN32_COMPILER_TLS)&&!defined(USE_COMPILER_TLS)&&!defined(USE_CUSTOM_SPECIFIC) #if defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32) #if defined(CYGWIN32)&&GC_GNUC_PREREQ(4,0) #if defined(__clang__) #define USE_PTHREAD_SPECIFIC #else #define USE_COMPILER_TLS #endif #elif defined(__GNUC__)||defined(MSWINCE) #define USE_WIN32_SPECIFIC #else #define USE_WIN32_COMPILER_TLS #endif #elif (defined(LINUX)&&!defined(ARM32)&&!defined(AVR32)&&GC_GNUC_PREREQ(3,3)&&!(defined(__clang__)&&defined(HOST_ANDROID)))||(defined(FREEBSD)||(defined(NETBSD)&&__NetBSD_Version__>=600000000)&&(GC_GNUC_PREREQ(4,4)||GC_CLANG_PREREQ(3,9)))||(defined(HOST_ANDROID)&&defined(ARM32)&&(GC_GNUC_PREREQ(4,6)||GC_CLANG_PREREQ_FULL(3,8,256229))) #define USE_COMPILER_TLS #elif defined(GC_DGUX386_THREADS)||defined(GC_OSF1_THREADS)||defined(GC_AIX_THREADS)||defined(GC_DARWIN_THREADS)||defined(GC_FREEBSD_THREADS)||defined(GC_NETBSD_THREADS)||defined(GC_LINUX_THREADS)||defined(GC_HAIKU_THREADS)||defined(GC_RTEMS_PTHREADS) #define USE_PTHREAD_SPECIFIC #elif defined(GC_HPUX_THREADS) #ifdef __GNUC__ #define USE_PTHREAD_SPECIFIC #else #define USE_COMPILER_TLS #endif #else #define USE_CUSTOM_SPECIFIC #endif #endif #ifndef THREAD_FREELISTS_KINDS #ifdef ENABLE_DISCLAIM #define THREAD_FREELISTS_KINDS (NORMAL+2) #else #define THREAD_FREELISTS_KINDS (NORMAL+1) #endif #endif typedef struct thread_local_freelists { void*_freelists[THREAD_FREELISTS_KINDS][TINY_FREELISTS]; #define ptrfree_freelists _freelists[PTRFREE] #define normal_freelists _freelists[NORMAL] #ifdef GC_GCJ_SUPPORT void*gcj_freelists[TINY_FREELISTS]; #define ERROR_FL ((void*)GC_WORD_MAX) #endif #define DIRECT_GRANULES (HBLKSIZE/GRANULE_BYTES) }*GC_tlfs; #if defined(USE_PTHREAD_SPECIFIC) #define GC_getspecific pthread_getspecific #define GC_setspecific pthread_setspecific #define GC_key_create pthread_key_create #define GC_remove_specific(key)pthread_setspecific(key,NULL) #define GC_remove_specific_after_fork(key,t)(void)0 typedef pthread_key_t GC_key_t; #elif defined(USE_COMPILER_TLS)||defined(USE_WIN32_COMPILER_TLS) #define GC_getspecific(x)(x) #define GC_setspecific(key,v)((key)=(v),0) #define GC_key_create(key,d)0 #define GC_remove_specific(key) #define GC_remove_specific_after_fork(key,t)(void)0 typedef void*GC_key_t; #elif defined(USE_WIN32_SPECIFIC) #define GC_getspecific TlsGetValue #define GC_setspecific(key,v)!TlsSetValue(key,v) #ifndef TLS_OUT_OF_INDEXES #define TLS_OUT_OF_INDEXES (DWORD)0xFFFFFFFF #endif #define GC_key_create(key,d)((d)!=0||(*(key)=TlsAlloc())==TLS_OUT_OF_INDEXES?-1:0) #define GC_remove_specific(key) #define GC_remove_specific_after_fork(key,t)(void)0 typedef DWORD GC_key_t; #elif defined(USE_CUSTOM_SPECIFIC) EXTERN_C_END #ifndef GC_SPECIFIC_H #define GC_SPECIFIC_H #include EXTERN_C_BEGIN #define MALLOC_CLEAR(n)GC_INTERNAL_MALLOC(n,NORMAL) #define TS_CACHE_SIZE 1024 #define CACHE_HASH(n)((((n)>>8)^(n))&(TS_CACHE_SIZE - 1)) #define TS_HASH_SIZE 1024 #define HASH(p)((unsigned)((((word)(p))>>8)^(word)(p))&(TS_HASH_SIZE - 1)) #ifdef GC_ASSERTIONS typedef GC_hidden_pointer ts_entry_value_t; #define TS_HIDE_VALUE(p)GC_HIDE_POINTER(p) #define TS_REVEAL_PTR(p)GC_REVEAL_POINTER(p) #else typedef void*ts_entry_value_t; #define TS_HIDE_VALUE(p)(p) #define TS_REVEAL_PTR(p)(p) #endif typedef struct thread_specific_entry { volatile AO_t qtid; ts_entry_value_t value; struct thread_specific_entry*next; pthread_t thread; } tse; #define quick_thread_id()(((word)GC_approx_sp())>>12) #define INVALID_QTID ((word)0) #define INVALID_THREADID ((pthread_t)0) union ptse_ao_u { tse*p; volatile AO_t ao; }; typedef struct thread_specific_data { tse*volatile cache[TS_CACHE_SIZE]; union ptse_ao_u hash[TS_HASH_SIZE]; pthread_mutex_t lock; } tsd; typedef tsd*GC_key_t; #define GC_key_create(key,d)GC_key_create_inner(key) GC_INNER int GC_key_create_inner(tsd**key_ptr); GC_INNER int GC_setspecific(tsd*key,void*value); #define GC_remove_specific(key)GC_remove_specific_after_fork(key,pthread_self()) GC_INNER void GC_remove_specific_after_fork(tsd*key,pthread_t t); GC_INNER void*GC_slow_getspecific(tsd*key,word qtid, tse*volatile*cache_entry); GC_INLINE void*GC_getspecific(tsd*key) { word qtid=quick_thread_id(); tse*volatile*entry_ptr=&key->cache[CACHE_HASH(qtid)]; tse*entry=*entry_ptr; GC_ASSERT(qtid!=INVALID_QTID); if (EXPECT(entry->qtid==qtid,TRUE)){ GC_ASSERT(entry->thread==pthread_self()); return TS_REVEAL_PTR(entry->value); } return GC_slow_getspecific(key,qtid,entry_ptr); } EXTERN_C_END #endif EXTERN_C_BEGIN #else #error implement me #endif GC_INNER void GC_init_thread_local(GC_tlfs p); GC_INNER void GC_destroy_thread_local(GC_tlfs p); GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p); #ifdef GC_ASSERTIONS GC_bool GC_is_thread_tsd_valid(void*tsd); void GC_check_tls_for(GC_tlfs p); #if defined(USE_CUSTOM_SPECIFIC) void GC_check_tsd_marks(tsd*key); #endif #endif #ifndef GC_ATTR_TLS_FAST #define GC_ATTR_TLS_FAST #endif extern #if defined(USE_COMPILER_TLS) __thread GC_ATTR_TLS_FAST #elif defined(USE_WIN32_COMPILER_TLS) __declspec(thread)GC_ATTR_TLS_FAST #endif GC_key_t GC_thread_key; EXTERN_C_END #endif #endif #include #if defined(USE_COMPILER_TLS) __thread GC_ATTR_TLS_FAST #elif defined(USE_WIN32_COMPILER_TLS) __declspec(thread)GC_ATTR_TLS_FAST #endif GC_key_t GC_thread_key; static GC_bool keys_initialized; static void return_single_freelist(void*fl,void**gfl) { if (*gfl==0){ *gfl=fl; } else { void*q,**qptr; GC_ASSERT(GC_size(fl)==GC_size(*gfl)); qptr=&(obj_link(fl)); while ((word)(q=*qptr)>=HBLKSIZE) qptr=&(obj_link(q)); GC_ASSERT(0==q); *qptr=*gfl; *gfl=fl; } } static void return_freelists(void**fl,void**gfl) { int i; for (i=1;i < TINY_FREELISTS;++i){ if ((word)(fl[i])>=HBLKSIZE){ return_single_freelist(fl[i],&gfl[i]); } fl[i]=(ptr_t)HBLKSIZE; } #ifdef GC_GCJ_SUPPORT if (fl[0]==ERROR_FL)return; #endif if ((word)(fl[0])>=HBLKSIZE){ return_single_freelist(fl[0],&gfl[1]); } } #ifdef USE_PTHREAD_SPECIFIC static void reset_thread_key(void*v){ pthread_setspecific(GC_thread_key,v); } #else #define reset_thread_key 0 #endif GC_INNER void GC_init_thread_local(GC_tlfs p) { int i,j,res; GC_ASSERT(I_HOLD_LOCK()); if (!EXPECT(keys_initialized,TRUE)){ GC_ASSERT((word)&GC_thread_key % sizeof(word)==0); res=GC_key_create(&GC_thread_key,reset_thread_key); if (COVERT_DATAFLOW(res)!=0){ ABORT("Failed to create key for local allocator"); } keys_initialized=TRUE; } res=GC_setspecific(GC_thread_key,p); if (COVERT_DATAFLOW(res)!=0){ ABORT("Failed to set thread specific allocation pointers"); } for (j=0;j < TINY_FREELISTS;++j){ for (i=0;i < THREAD_FREELISTS_KINDS;++i){ p->_freelists[i][j]=(void*)(word)1; } #ifdef GC_GCJ_SUPPORT p->gcj_freelists[j]=(void*)(word)1; #endif } #ifdef GC_GCJ_SUPPORT p->gcj_freelists[0]=ERROR_FL; #endif } GC_INNER void GC_destroy_thread_local(GC_tlfs p) { int k; GC_STATIC_ASSERT(THREAD_FREELISTS_KINDS<=MAXOBJKINDS); for (k=0;k < THREAD_FREELISTS_KINDS;++k){ if (k==(int)GC_n_kinds) break; return_freelists(p->_freelists[k],GC_obj_kinds[k].ok_freelist); } #ifdef GC_GCJ_SUPPORT return_freelists(p->gcj_freelists,(void**)GC_gcjobjfreelist); #endif } GC_API GC_ATTR_MALLOC void*GC_CALL GC_malloc_kind(size_t bytes,int kind) { size_t granules; void*tsd; void*result; #if MAXOBJKINDS > THREAD_FREELISTS_KINDS if (EXPECT(kind>=THREAD_FREELISTS_KINDS,FALSE)){ return GC_malloc_kind_global(bytes,kind); } #endif #if!defined(USE_PTHREAD_SPECIFIC)&&!defined(USE_WIN32_SPECIFIC) { GC_key_t k=GC_thread_key; if (EXPECT(0==k,FALSE)){ return GC_malloc_kind_global(bytes,kind); } tsd=GC_getspecific(k); } #else if (!EXPECT(keys_initialized,TRUE)) return GC_malloc_kind_global(bytes,kind); tsd=GC_getspecific(GC_thread_key); #endif #if!defined(USE_COMPILER_TLS)&&!defined(USE_WIN32_COMPILER_TLS) if (EXPECT(0==tsd,FALSE)){ return GC_malloc_kind_global(bytes,kind); } #endif GC_ASSERT(GC_is_initialized); GC_ASSERT(GC_is_thread_tsd_valid(tsd)); granules=ROUNDED_UP_GRANULES(bytes); #if defined(CPPCHECK) #define MALLOC_KIND_PTRFREE_INIT (void*)1 #else #define MALLOC_KIND_PTRFREE_INIT NULL #endif GC_FAST_MALLOC_GRANS(result,granules, ((GC_tlfs)tsd)->_freelists[kind],DIRECT_GRANULES, kind,GC_malloc_kind_global(bytes,kind), (void)(kind==PTRFREE?MALLOC_KIND_PTRFREE_INIT :(obj_link(result)=0))); #ifdef LOG_ALLOCS GC_log_printf("GC_malloc_kind(%lu,%d)returned %p,recent GC #%lu\n", (unsigned long)bytes,kind,result, (unsigned long)GC_gc_no); #endif return result; } #ifdef GC_GCJ_SUPPORT GC_API GC_ATTR_MALLOC void*GC_CALL GC_gcj_malloc(size_t bytes, void*ptr_to_struct_containing_descr) { if (EXPECT(GC_incremental,FALSE)){ return GC_core_gcj_malloc(bytes,ptr_to_struct_containing_descr); } else { size_t granules=ROUNDED_UP_GRANULES(bytes); void*result; void**tiny_fl; GC_ASSERT(GC_gcjobjfreelist!=NULL); tiny_fl=((GC_tlfs)GC_getspecific(GC_thread_key))->gcj_freelists; GC_FAST_MALLOC_GRANS(result,granules,tiny_fl,DIRECT_GRANULES, GC_gcj_kind, GC_core_gcj_malloc(bytes, ptr_to_struct_containing_descr), {AO_compiler_barrier(); *(void**)result=ptr_to_struct_containing_descr;}); return result; } } #endif GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p) { ptr_t q; int i,j; for (j=0;j < TINY_FREELISTS;++j){ for (i=0;i < THREAD_FREELISTS_KINDS;++i){ q=(ptr_t)AO_load((volatile AO_t*)&p->_freelists[i][j]); if ((word)q > HBLKSIZE) GC_set_fl_marks(q); } #ifdef GC_GCJ_SUPPORT if (EXPECT(j > 0,TRUE)){ q=(ptr_t)AO_load((volatile AO_t*)&p->gcj_freelists[j]); if ((word)q > HBLKSIZE) GC_set_fl_marks(q); } #endif } } #if defined(GC_ASSERTIONS) void GC_check_tls_for(GC_tlfs p) { int i,j; for (j=1;j < TINY_FREELISTS;++j){ for (i=0;i < THREAD_FREELISTS_KINDS;++i){ GC_check_fl_marks(&p->_freelists[i][j]); } #ifdef GC_GCJ_SUPPORT GC_check_fl_marks(&p->gcj_freelists[j]); #endif } } #endif #endif #ifndef GC_PTHREAD_SUPPORT_H #define GC_PTHREAD_SUPPORT_H #if defined(GC_PTHREADS)&&!defined(GC_WIN32_THREADS) #if defined(GC_DARWIN_THREADS) #else #ifndef GC_PTHREAD_STOP_WORLD_H #define GC_PTHREAD_STOP_WORLD_H EXTERN_C_BEGIN struct thread_stop_info { #if!defined(GC_OPENBSD_UTHREADS)&&!defined(NACL)&&!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PSP2) volatile AO_t last_stop_count; #endif ptr_t stack_ptr; #ifdef NACL #ifdef ARM32 #define NACL_GC_REG_STORAGE_SIZE 9 #else #define NACL_GC_REG_STORAGE_SIZE 20 #endif ptr_t reg_storage[NACL_GC_REG_STORAGE_SIZE]; #elif defined(SN_TARGET_ORBIS) #define ORBIS_GC_REG_STORAGE_SIZE 27 word registers[ORBIS_GC_REG_STORAGE_SIZE]; #endif }; GC_INNER void GC_stop_init(void); EXTERN_C_END #endif #endif #ifdef THREAD_LOCAL_ALLOC #endif #ifdef THREAD_SANITIZER #endif EXTERN_C_BEGIN typedef struct GC_Thread_Rep { #ifdef THREAD_SANITIZER char dummy[sizeof(oh)]; #endif struct GC_Thread_Rep*next; pthread_t id; #ifdef USE_TKILL_ON_ANDROID pid_t kernel_id; #endif struct thread_stop_info stop_info; #if defined(GC_ENABLE_SUSPEND_THREAD)&&!defined(GC_DARWIN_THREADS)&&!defined(GC_OPENBSD_UTHREADS)&&!defined(NACL) volatile AO_t suspended_ext; #endif unsigned char flags; #define FINISHED 1 #define DETACHED 2 #define MAIN_THREAD 4 #define DISABLED_GC 0x10 unsigned char thread_blocked; unsigned short finalizer_skipped; unsigned char finalizer_nested; ptr_t stack_end; ptr_t altstack; word altstack_size; ptr_t stack; word stack_size; #if defined(GC_DARWIN_THREADS)&&!defined(DARWIN_DONT_PARSE_STACK) ptr_t topOfStack; #endif #ifdef IA64 ptr_t backing_store_end; ptr_t backing_store_ptr; #endif struct GC_traced_stack_sect_s*traced_stack_sect; void*status; #ifdef THREAD_LOCAL_ALLOC struct thread_local_freelists tlfs GC_ATTR_WORD_ALIGNED; #endif }*GC_thread; #ifndef THREAD_TABLE_SZ #define THREAD_TABLE_SZ 256 #endif #if CPP_WORDSZ==64 #define THREAD_TABLE_INDEX(id)(int)(((((NUMERIC_THREAD_ID(id)>>8)^NUMERIC_THREAD_ID(id))>>16)^((NUMERIC_THREAD_ID(id)>>8)^NUMERIC_THREAD_ID(id)))% THREAD_TABLE_SZ) #else #define THREAD_TABLE_INDEX(id)(int)(((NUMERIC_THREAD_ID(id)>>16)^(NUMERIC_THREAD_ID(id)>>8)^NUMERIC_THREAD_ID(id))% THREAD_TABLE_SZ) #endif GC_EXTERN volatile GC_thread GC_threads[THREAD_TABLE_SZ]; GC_EXTERN GC_bool GC_thr_initialized; GC_INNER GC_thread GC_lookup_thread(pthread_t id); #ifdef NACL GC_EXTERN __thread GC_thread GC_nacl_gc_thread_self; GC_INNER void GC_nacl_initialize_gc_thread(void); GC_INNER void GC_nacl_shutdown_gc_thread(void); #endif #ifdef GC_EXPLICIT_SIGNALS_UNBLOCK GC_INNER void GC_unblock_gc_signals(void); #endif #ifdef GC_PTHREAD_START_STANDALONE #define GC_INNER_PTHRSTART #else #define GC_INNER_PTHRSTART GC_INNER #endif GC_INNER_PTHRSTART void*GC_CALLBACK GC_inner_start_routine( struct GC_stack_base*sb,void*arg); GC_INNER_PTHRSTART GC_thread GC_start_rtn_prepare_thread( void*(**pstart)(void*), void**pstart_arg, struct GC_stack_base*sb,void*arg); GC_INNER_PTHRSTART void GC_thread_exit_proc(void*); EXTERN_C_END #endif #endif #if defined(GC_DARWIN_THREADS) #include #include #include #ifdef POWERPC #if CPP_WORDSZ==32 #define PPC_RED_ZONE_SIZE 224 #elif CPP_WORDSZ==64 #define PPC_RED_ZONE_SIZE 320 #endif #endif #ifndef DARWIN_DONT_PARSE_STACK typedef struct StackFrame { unsigned long savedSP; unsigned long savedCR; unsigned long savedLR; unsigned long reserved[2]; unsigned long savedRTOC; } StackFrame; GC_INNER ptr_t GC_FindTopOfStack(unsigned long stack_start) { StackFrame*frame=(StackFrame*)stack_start; if (stack_start==0){ #ifdef POWERPC #if CPP_WORDSZ==32 __asm__ __volatile__ ("lwz %0,0(r1)":"=r" (frame)); #else __asm__ __volatile__ ("ld %0,0(r1)":"=r" (frame)); #endif #elif defined(ARM32) volatile ptr_t sp_reg; __asm__ __volatile__ ("mov %0,r7\n":"=r" (sp_reg)); frame=(StackFrame*)sp_reg; #elif defined(AARCH64) volatile ptr_t sp_reg; __asm__ __volatile__ ("mov %0,x29\n":"=r" (sp_reg)); frame=(StackFrame*)sp_reg; #else #if defined(CPPCHECK) GC_noop1((word)&frame); #endif ABORT("GC_FindTopOfStack(0)is not implemented"); #endif } #ifdef DEBUG_THREADS_EXTRA GC_log_printf("FindTopOfStack start at sp=%p\n",(void*)frame); #endif while (frame->savedSP!=0){ frame=(StackFrame*)frame->savedSP; if ((frame->savedLR&~0x3)==0||(frame->savedLR&~0x3)==~0x3UL) break; } #ifdef DEBUG_THREADS_EXTRA GC_log_printf("FindTopOfStack finish at sp=%p\n",(void*)frame); #endif return (ptr_t)frame; } #endif #ifdef GC_NO_THREADS_DISCOVERY #define GC_query_task_threads FALSE #elif defined(GC_DISCOVER_TASK_THREADS) #define GC_query_task_threads TRUE #else STATIC GC_bool GC_query_task_threads=FALSE; #endif GC_API void GC_CALL GC_use_threads_discovery(void) { #if defined(GC_NO_THREADS_DISCOVERY)||defined(DARWIN_DONT_PARSE_STACK) ABORT("Darwin task-threads-based stop and push unsupported"); #else #ifndef GC_ALWAYS_MULTITHREADED GC_ASSERT(!GC_need_to_lock); #endif #ifndef GC_DISCOVER_TASK_THREADS GC_query_task_threads=TRUE; #endif GC_init_parallel(); #endif } #ifndef kCFCoreFoundationVersionNumber_iOS_8_0 #define kCFCoreFoundationVersionNumber_iOS_8_0 1140.1 #endif STATIC ptr_t GC_stack_range_for(ptr_t*phi,thread_act_t thread,GC_thread p, GC_bool thread_blocked,mach_port_t my_thread, ptr_t*paltstack_lo, ptr_t*paltstack_hi GC_ATTR_UNUSED) { ptr_t lo; if (thread==my_thread){ GC_ASSERT(!thread_blocked); lo=GC_approx_sp(); #ifndef DARWIN_DONT_PARSE_STACK *phi=GC_FindTopOfStack(0); #endif } else if (thread_blocked){ #if defined(CPPCHECK) if (NULL==p)ABORT("Invalid GC_thread passed to GC_stack_range_for"); #endif lo=p->stop_info.stack_ptr; #ifndef DARWIN_DONT_PARSE_STACK *phi=p->topOfStack; #endif } else { kern_return_t kern_result; GC_THREAD_STATE_T state; #if defined(ARM32)&&defined(ARM_THREAD_STATE32) size_t size; static cpu_type_t cputype=0; if (cputype==0){ sysctlbyname("hw.cputype",&cputype,&size,NULL,0); } if (cputype==CPU_TYPE_ARM64 ||kCFCoreFoundationVersionNumber >=kCFCoreFoundationVersionNumber_iOS_8_0){ arm_unified_thread_state_t unified_state; mach_msg_type_number_t unified_thread_state_count =ARM_UNIFIED_THREAD_STATE_COUNT; #if defined(CPPCHECK) #define GC_ARM_UNIFIED_THREAD_STATE 1 #else #define GC_ARM_UNIFIED_THREAD_STATE ARM_UNIFIED_THREAD_STATE #endif kern_result=thread_get_state(thread,GC_ARM_UNIFIED_THREAD_STATE, (natural_t*)&unified_state, &unified_thread_state_count); #if!defined(CPPCHECK) if (unified_state.ash.flavor!=ARM_THREAD_STATE32){ ABORT("unified_state flavor should be ARM_THREAD_STATE32"); } #endif state=unified_state; } else #endif { mach_msg_type_number_t thread_state_count=GC_MACH_THREAD_STATE_COUNT; kern_result=thread_get_state(thread,GC_MACH_THREAD_STATE, (natural_t*)&state, &thread_state_count); } #ifdef DEBUG_THREADS GC_log_printf("thread_get_state returns value=%d\n",kern_result); #endif if (kern_result!=KERN_SUCCESS) ABORT("thread_get_state failed"); #if defined(I386) lo=(ptr_t)state.THREAD_FLD(esp); #ifndef DARWIN_DONT_PARSE_STACK *phi=GC_FindTopOfStack(state.THREAD_FLD(esp)); #endif GC_push_one(state.THREAD_FLD(eax)); GC_push_one(state.THREAD_FLD(ebx)); GC_push_one(state.THREAD_FLD(ecx)); GC_push_one(state.THREAD_FLD(edx)); GC_push_one(state.THREAD_FLD(edi)); GC_push_one(state.THREAD_FLD(esi)); GC_push_one(state.THREAD_FLD(ebp)); #elif defined(X86_64) lo=(ptr_t)state.THREAD_FLD(rsp); #ifndef DARWIN_DONT_PARSE_STACK *phi=GC_FindTopOfStack(state.THREAD_FLD(rsp)); #endif GC_push_one(state.THREAD_FLD(rax)); GC_push_one(state.THREAD_FLD(rbx)); GC_push_one(state.THREAD_FLD(rcx)); GC_push_one(state.THREAD_FLD(rdx)); GC_push_one(state.THREAD_FLD(rdi)); GC_push_one(state.THREAD_FLD(rsi)); GC_push_one(state.THREAD_FLD(rbp)); GC_push_one(state.THREAD_FLD(r8)); GC_push_one(state.THREAD_FLD(r9)); GC_push_one(state.THREAD_FLD(r10)); GC_push_one(state.THREAD_FLD(r11)); GC_push_one(state.THREAD_FLD(r12)); GC_push_one(state.THREAD_FLD(r13)); GC_push_one(state.THREAD_FLD(r14)); GC_push_one(state.THREAD_FLD(r15)); #elif defined(POWERPC) lo=(ptr_t)(state.THREAD_FLD(r1)- PPC_RED_ZONE_SIZE); #ifndef DARWIN_DONT_PARSE_STACK *phi=GC_FindTopOfStack(state.THREAD_FLD(r1)); #endif GC_push_one(state.THREAD_FLD(r0)); GC_push_one(state.THREAD_FLD(r2)); GC_push_one(state.THREAD_FLD(r3)); GC_push_one(state.THREAD_FLD(r4)); GC_push_one(state.THREAD_FLD(r5)); GC_push_one(state.THREAD_FLD(r6)); GC_push_one(state.THREAD_FLD(r7)); GC_push_one(state.THREAD_FLD(r8)); GC_push_one(state.THREAD_FLD(r9)); GC_push_one(state.THREAD_FLD(r10)); GC_push_one(state.THREAD_FLD(r11)); GC_push_one(state.THREAD_FLD(r12)); GC_push_one(state.THREAD_FLD(r13)); GC_push_one(state.THREAD_FLD(r14)); GC_push_one(state.THREAD_FLD(r15)); GC_push_one(state.THREAD_FLD(r16)); GC_push_one(state.THREAD_FLD(r17)); GC_push_one(state.THREAD_FLD(r18)); GC_push_one(state.THREAD_FLD(r19)); GC_push_one(state.THREAD_FLD(r20)); GC_push_one(state.THREAD_FLD(r21)); GC_push_one(state.THREAD_FLD(r22)); GC_push_one(state.THREAD_FLD(r23)); GC_push_one(state.THREAD_FLD(r24)); GC_push_one(state.THREAD_FLD(r25)); GC_push_one(state.THREAD_FLD(r26)); GC_push_one(state.THREAD_FLD(r27)); GC_push_one(state.THREAD_FLD(r28)); GC_push_one(state.THREAD_FLD(r29)); GC_push_one(state.THREAD_FLD(r30)); GC_push_one(state.THREAD_FLD(r31)); #elif defined(ARM32) lo=(ptr_t)state.THREAD_FLD(sp); #ifndef DARWIN_DONT_PARSE_STACK *phi=GC_FindTopOfStack(state.THREAD_FLD(r[7])); #endif { int j; for (j=0;j < 7;j++) GC_push_one(state.THREAD_FLD(r[j])); j++; for (;j<=12;j++) GC_push_one(state.THREAD_FLD(r[j])); } GC_push_one(state.THREAD_FLD(lr)); #elif defined(AARCH64) lo=(ptr_t)state.THREAD_FLD(sp); #ifndef DARWIN_DONT_PARSE_STACK *phi=GC_FindTopOfStack(state.THREAD_FLD(fp)); #endif { int j; for (j=0;j<=28;j++){ GC_push_one(state.THREAD_FLD(x[j])); } } GC_push_one(state.THREAD_FLD(lr)); #elif defined(CPPCHECK) lo=NULL; #else #error FIXME for non-arm/ppc/x86 architectures #endif } #ifdef DARWIN_DONT_PARSE_STACK *phi=(p->flags&MAIN_THREAD)!=0?GC_stackbottom:p->stack_end; #endif #ifdef DARWIN_DONT_PARSE_STACK if (p->altstack!=NULL&&(word)p->altstack<=(word)lo &&(word)lo<=(word)p->altstack+p->altstack_size){ *paltstack_lo=lo; *paltstack_hi=p->altstack+p->altstack_size; lo=p->stack; *phi=p->stack+p->stack_size; } else #endif { *paltstack_lo=NULL; } #ifdef DEBUG_THREADS GC_log_printf("Darwin:Stack for thread %p=[%p,%p)\n", (void*)(word)thread,(void*)lo,(void*)(*phi)); #endif return lo; } GC_INNER void GC_push_all_stacks(void) { ptr_t hi,altstack_lo,altstack_hi; task_t my_task=current_task(); mach_port_t my_thread=mach_thread_self(); GC_bool found_me=FALSE; int nthreads=0; word total_size=0; mach_msg_type_number_t listcount=(mach_msg_type_number_t)THREAD_TABLE_SZ; if (!EXPECT(GC_thr_initialized,TRUE)) GC_thr_init(); #ifndef DARWIN_DONT_PARSE_STACK if (GC_query_task_threads){ int i; kern_return_t kern_result; thread_act_array_t act_list=0; kern_result=task_threads(my_task,&act_list,&listcount); if (kern_result!=KERN_SUCCESS) ABORT("task_threads failed"); for (i=0;i < (int)listcount;i++){ thread_act_t thread=act_list[i]; ptr_t lo=GC_stack_range_for(&hi,thread,NULL,FALSE,my_thread, &altstack_lo,&altstack_hi); if (lo){ GC_ASSERT((word)lo<=(word)hi); total_size+=hi - lo; GC_push_all_stack(lo,hi); } nthreads++; if (thread==my_thread) found_me=TRUE; mach_port_deallocate(my_task,thread); } vm_deallocate(my_task,(vm_address_t)act_list, sizeof(thread_t)*listcount); } else #endif { int i; for (i=0;i < (int)listcount;i++){ GC_thread p; for (p=GC_threads[i];p!=NULL;p=p->next) if ((p->flags&FINISHED)==0){ thread_act_t thread=(thread_act_t)p->stop_info.mach_thread; ptr_t lo=GC_stack_range_for(&hi,thread,p, (GC_bool)p->thread_blocked, my_thread,&altstack_lo, &altstack_hi); if (lo){ GC_ASSERT((word)lo<=(word)hi); total_size+=hi - lo; GC_push_all_stack_sections(lo,hi,p->traced_stack_sect); } if (altstack_lo){ total_size+=altstack_hi - altstack_lo; GC_push_all_stack(altstack_lo,altstack_hi); } nthreads++; if (thread==my_thread) found_me=TRUE; } } } mach_port_deallocate(my_task,my_thread); GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks\n",nthreads); if (!found_me&&!GC_in_thread_creation) ABORT("Collecting from unknown thread"); GC_total_stacksize=total_size; } #ifndef GC_NO_THREADS_DISCOVERY #ifdef MPROTECT_VDB STATIC mach_port_t GC_mach_handler_thread=0; STATIC GC_bool GC_use_mach_handler_thread=FALSE; GC_INNER void GC_darwin_register_mach_handler_thread(mach_port_t thread) { GC_mach_handler_thread=thread; GC_use_mach_handler_thread=TRUE; } #endif #ifndef GC_MAX_MACH_THREADS #define GC_MAX_MACH_THREADS THREAD_TABLE_SZ #endif struct GC_mach_thread { thread_act_t thread; GC_bool suspended; }; struct GC_mach_thread GC_mach_threads[GC_MAX_MACH_THREADS]; STATIC int GC_mach_threads_count=0; STATIC GC_bool GC_suspend_thread_list(thread_act_array_t act_list,int count, thread_act_array_t old_list, int old_count,task_t my_task, mach_port_t my_thread) { int i; int j=-1; GC_bool changed=FALSE; for (i=0;i < count;i++){ thread_act_t thread=act_list[i]; GC_bool found; kern_return_t kern_result; if (thread==my_thread #ifdef MPROTECT_VDB ||(GC_mach_handler_thread==thread&&GC_use_mach_handler_thread) #endif #ifdef PARALLEL_MARK ||GC_is_mach_marker(thread) #endif ){ mach_port_deallocate(my_task,thread); continue; } found=FALSE; { int last_found=j; while (++j < old_count) if (old_list[j]==thread){ found=TRUE; break; } if (!found){ for (j=0;j < last_found;j++) if (old_list[j]==thread){ found=TRUE; break; } } } if (found){ mach_port_deallocate(my_task,thread); continue; } if (GC_mach_threads_count==GC_MAX_MACH_THREADS) ABORT("Too many threads"); GC_mach_threads[GC_mach_threads_count].thread=thread; GC_mach_threads[GC_mach_threads_count].suspended=FALSE; changed=TRUE; #ifdef DEBUG_THREADS GC_log_printf("Suspending %p\n",(void*)(word)thread); #endif GC_acquire_dirty_lock(); do { kern_result=thread_suspend(thread); } while (kern_result==KERN_ABORTED); GC_release_dirty_lock(); if (kern_result!=KERN_SUCCESS){ GC_mach_threads[GC_mach_threads_count].suspended=FALSE; } else { GC_mach_threads[GC_mach_threads_count].suspended=TRUE; if (GC_on_thread_event) GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,(void*)(word)thread); } GC_mach_threads_count++; } return changed; } #endif GC_INNER void GC_stop_world(void) { task_t my_task=current_task(); mach_port_t my_thread=mach_thread_self(); kern_return_t kern_result; #ifdef DEBUG_THREADS GC_log_printf("Stopping the world from thread %p\n", (void*)(word)my_thread); #endif #ifdef PARALLEL_MARK if (GC_parallel){ GC_acquire_mark_lock(); GC_ASSERT(GC_fl_builder_count==0); } #endif if (GC_query_task_threads){ #ifndef GC_NO_THREADS_DISCOVERY GC_bool changed; thread_act_array_t act_list,prev_list; mach_msg_type_number_t listcount,prevcount; GC_mach_threads_count=0; changed=TRUE; prev_list=NULL; prevcount=0; do { kern_result=task_threads(my_task,&act_list,&listcount); if (kern_result==KERN_SUCCESS){ changed=GC_suspend_thread_list(act_list,listcount,prev_list, prevcount,my_task,my_thread); if (prev_list!=NULL){ vm_deallocate(my_task,(vm_address_t)prev_list, sizeof(thread_t)*prevcount); } prev_list=act_list; prevcount=listcount; } } while (changed); GC_ASSERT(prev_list!=0); vm_deallocate(my_task,(vm_address_t)act_list, sizeof(thread_t)*listcount); #endif } else { unsigned i; for (i=0;i < THREAD_TABLE_SZ;i++){ GC_thread p; for (p=GC_threads[i];p!=NULL;p=p->next){ if ((p->flags&FINISHED)==0&&!p->thread_blocked&& p->stop_info.mach_thread!=my_thread){ GC_acquire_dirty_lock(); do { kern_result=thread_suspend(p->stop_info.mach_thread); } while (kern_result==KERN_ABORTED); GC_release_dirty_lock(); if (kern_result!=KERN_SUCCESS) ABORT("thread_suspend failed"); if (GC_on_thread_event) GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, (void*)(word)p->stop_info.mach_thread); } } } } #ifdef MPROTECT_VDB if (GC_auto_incremental){ GC_mprotect_stop(); } #endif #ifdef PARALLEL_MARK if (GC_parallel) GC_release_mark_lock(); #endif #ifdef DEBUG_THREADS GC_log_printf("World stopped from %p\n",(void*)(word)my_thread); #endif mach_port_deallocate(my_task,my_thread); } GC_INLINE void GC_thread_resume(thread_act_t thread) { kern_return_t kern_result; #if defined(DEBUG_THREADS)||defined(GC_ASSERTIONS) struct thread_basic_info info; mach_msg_type_number_t outCount=THREAD_BASIC_INFO_COUNT; #if defined(CPPCHECK)&&defined(DEBUG_THREADS) info.run_state=0; #endif kern_result=thread_info(thread,THREAD_BASIC_INFO, (thread_info_t)&info,&outCount); if (kern_result!=KERN_SUCCESS) ABORT("thread_info failed"); #endif #ifdef DEBUG_THREADS GC_log_printf("Resuming thread %p with state %d\n",(void*)(word)thread, info.run_state); #endif kern_result=thread_resume(thread); if (kern_result!=KERN_SUCCESS){ WARN("thread_resume(%p)failed:mach port invalid\n",thread); } else if (GC_on_thread_event){ GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED,(void*)(word)thread); } } GC_INNER void GC_start_world(void) { task_t my_task=current_task(); #ifdef DEBUG_THREADS GC_log_printf("World starting\n"); #endif #ifdef MPROTECT_VDB if (GC_auto_incremental){ GC_mprotect_resume(); } #endif if (GC_query_task_threads){ #ifndef GC_NO_THREADS_DISCOVERY int i,j; kern_return_t kern_result; thread_act_array_t act_list; mach_msg_type_number_t listcount; kern_result=task_threads(my_task,&act_list,&listcount); if (kern_result!=KERN_SUCCESS) ABORT("task_threads failed"); j=(int)listcount; for (i=0;i < GC_mach_threads_count;i++){ thread_act_t thread=GC_mach_threads[i].thread; if (GC_mach_threads[i].suspended){ int last_found=j; while (++j < (int)listcount){ if (act_list[j]==thread) break; } if (j>=(int)listcount){ for (j=0;j < last_found;j++){ if (act_list[j]==thread) break; } } if (j!=last_found){ GC_thread_resume(thread); } } else { #ifdef DEBUG_THREADS GC_log_printf("Not resuming thread %p as it is not suspended\n", (void*)(word)thread); #endif } mach_port_deallocate(my_task,thread); } for (i=0;i < (int)listcount;i++) mach_port_deallocate(my_task,act_list[i]); vm_deallocate(my_task,(vm_address_t)act_list, sizeof(thread_t)*listcount); #endif } else { int i; mach_port_t my_thread=mach_thread_self(); for (i=0;i < THREAD_TABLE_SZ;i++){ GC_thread p; for (p=GC_threads[i];p!=NULL;p=p->next){ if ((p->flags&FINISHED)==0&&!p->thread_blocked&& p->stop_info.mach_thread!=my_thread) GC_thread_resume(p->stop_info.mach_thread); } } mach_port_deallocate(my_task,my_thread); } #ifdef DEBUG_THREADS GC_log_printf("World started\n"); #endif } #endif #if!defined(MACOS)&&!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PSP2)&&!defined(_WIN32_WCE)&&!defined(__CC_ARM) #include #endif #undef GC_MUST_RESTORE_REDEFINED_DLOPEN #if defined(GC_PTHREADS)&&!defined(GC_NO_DLOPEN)&&!defined(GC_NO_THREAD_REDIRECTS)&&!defined(GC_USE_LD_WRAP) #undef dlopen #define GC_MUST_RESTORE_REDEFINED_DLOPEN #endif STATIC GC_has_static_roots_func GC_has_static_roots=0; #if (defined(DYNAMIC_LOADING)||defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32))&&!defined(PCR) #if!defined(DARWIN)&&!defined(SCO_ELF)&&!defined(SOLARISDL)&&!defined(AIX)&&!defined(DGUX)&&!defined(IRIX5)&&!defined(HPUX)&&!defined(CYGWIN32)&&!defined(MSWIN32)&&!defined(MSWINCE)&&!(defined(ALPHA)&&defined(OSF1))&&!(defined(FREEBSD)&&defined(__ELF__))&&!(defined(LINUX)&&defined(__ELF__))&&!(defined(NETBSD)&&defined(__ELF__))&&!(defined(OPENBSD)&&(defined(__ELF__)||defined(M68K)))&&!defined(HAIKU)&&!defined(HURD)&&!defined(NACL)&&!defined(CPPCHECK) #error We only know how to find data segments of dynamic libraries for above. #error Additional SVR4 variants might not be too hard to add. #endif #include #ifdef SOLARISDL #include #include #include #endif #if defined(NETBSD) #include #include #include #define ELFSIZE ARCH_ELFSIZE #endif #if defined(OPENBSD) #include #if (OpenBSD>=200519)&&!defined(HAVE_DL_ITERATE_PHDR) #define HAVE_DL_ITERATE_PHDR #endif #endif #if defined(SCO_ELF)||defined(DGUX)||defined(HURD)||defined(NACL)||(defined(__ELF__)&&(defined(LINUX)||defined(FREEBSD)||defined(NETBSD)||defined(OPENBSD))) #include #if!defined(OPENBSD)&&!defined(HOST_ANDROID) #include #endif #ifdef HOST_ANDROID #ifdef BIONIC_ELFDATA_REDEF_BUG #include #include #undef ELF_DATA #undef EM_ALPHA #endif #include #if!defined(GC_DONT_DEFINE_LINK_MAP)&&!(__ANDROID_API__>=21) struct link_map { uintptr_t l_addr; char*l_name; uintptr_t l_ld; struct link_map*l_next; struct link_map*l_prev; }; struct r_debug { int32_t r_version; struct link_map*r_map; void (*r_brk)(void); int32_t r_state; uintptr_t r_ldbase; }; #endif #else EXTERN_C_BEGIN #include EXTERN_C_END #endif #endif #ifndef ElfW #if defined(FREEBSD) #if __ELF_WORD_SIZE==32 #define ElfW(type)Elf32_##type #else #define ElfW(type)Elf64_##type #endif #elif defined(NETBSD)||defined(OPENBSD) #if ELFSIZE==32 #define ElfW(type)Elf32_##type #elif ELFSIZE==64 #define ElfW(type)Elf64_##type #else #error Missing ELFSIZE define #endif #else #if!defined(ELF_CLASS)||ELF_CLASS==ELFCLASS32 #define ElfW(type)Elf32_##type #else #define ElfW(type)Elf64_##type #endif #endif #endif #if defined(SOLARISDL)&&!defined(USE_PROC_FOR_LIBRARIES) EXTERN_C_BEGIN extern ElfW(Dyn)_DYNAMIC; EXTERN_C_END STATIC struct link_map* GC_FirstDLOpenedLinkMap(void) { ElfW(Dyn)*dp; static struct link_map*cachedResult=0; static ElfW(Dyn)*dynStructureAddr=0; #ifdef SUNOS53_SHARED_LIB if( dynStructureAddr==0){ void*startupSyms=dlopen(0,RTLD_LAZY); dynStructureAddr=(ElfW(Dyn)*)(word)dlsym(startupSyms,"_DYNAMIC"); } #else dynStructureAddr=&_DYNAMIC; #endif if (0==COVERT_DATAFLOW(dynStructureAddr)){ return(0); } if (cachedResult==0){ int tag; for( dp=((ElfW(Dyn)*)(&_DYNAMIC));(tag=dp->d_tag)!=0;dp++){ if (tag==DT_DEBUG){ struct r_debug*rd=(struct r_debug*)dp->d_un.d_ptr; if (rd!=NULL){ struct link_map*lm=rd->r_map; if (lm!=NULL) cachedResult=lm->l_next; } break; } } } return cachedResult; } #endif #ifdef GC_MUST_RESTORE_REDEFINED_DLOPEN #define dlopen GC_dlopen #endif #if defined(SOLARISDL) #if!defined(PCR)&&!defined(GC_SOLARIS_THREADS)&&defined(THREADS)&&!defined(CPPCHECK) #error Fix mutual exclusion with dlopen #endif #ifndef USE_PROC_FOR_LIBRARIES GC_INNER void GC_register_dynamic_libraries(void) { struct link_map*lm; for (lm=GC_FirstDLOpenedLinkMap();lm!=0;lm=lm->l_next){ ElfW(Ehdr)*e; ElfW(Phdr)*p; unsigned long offset; char*start; int i; e=(ElfW(Ehdr)*)lm->l_addr; p=((ElfW(Phdr)*)(((char*)(e))+e->e_phoff)); offset=((unsigned long)(lm->l_addr)); for( i=0;i < (int)e->e_phnum;i++,p++){ switch( p->p_type){ case PT_LOAD: { if(!(p->p_flags&PF_W))break; start=((char*)(p->p_vaddr))+offset; GC_add_roots_inner(start,start+p->p_memsz,TRUE); } break; default: break; } } } } #endif #endif #if defined(SCO_ELF)||defined(DGUX)||defined(HURD)||defined(NACL)||(defined(__ELF__)&&(defined(LINUX)||defined(FREEBSD)||defined(NETBSD)||defined(OPENBSD))) #ifdef USE_PROC_FOR_LIBRARIES #include #include #include #include #define MAPS_BUF_SIZE (32*1024) static void sort_heap_sects(struct HeapSect*base,size_t number_of_elements) { signed_word n=(signed_word)number_of_elements; signed_word nsorted=1; while (nsorted < n){ signed_word i; while (nsorted < n&& (word)base[nsorted-1].hs_start < (word)base[nsorted].hs_start) ++nsorted; if (nsorted==n)break; GC_ASSERT((word)base[nsorted-1].hs_start > (word)base[nsorted].hs_start); i=nsorted - 1; while (i>=0&&(word)base[i].hs_start > (word)base[i+1].hs_start){ struct HeapSect tmp=base[i]; base[i]=base[i+1]; base[i+1]=tmp; --i; } GC_ASSERT((word)base[nsorted-1].hs_start < (word)base[nsorted].hs_start); ++nsorted; } } STATIC void GC_register_map_entries(char*maps) { char*prot; char*buf_ptr=maps; ptr_t start,end; unsigned int maj_dev; ptr_t least_ha,greatest_ha; unsigned i; GC_ASSERT(I_HOLD_LOCK()); sort_heap_sects(GC_our_memory,GC_n_memory); least_ha=GC_our_memory[0].hs_start; greatest_ha=GC_our_memory[GC_n_memory-1].hs_start +GC_our_memory[GC_n_memory-1].hs_bytes; for (;;){ buf_ptr=GC_parse_map_entry(buf_ptr,&start,&end,&prot, &maj_dev,0); if (NULL==buf_ptr) break; if (prot[1]=='w'){ if ((word)start<=(word)GC_stackbottom &&(word)end>=(word)GC_stackbottom){ continue; } #ifdef THREADS if (GC_segment_is_thread_stack(start,end))continue; #endif if ((word)end<=(word)least_ha ||(word)start>=(word)greatest_ha){ GC_add_roots_inner(start,end,TRUE); continue; } i=0; while ((word)(GC_our_memory[i].hs_start +GC_our_memory[i].hs_bytes)< (word)start) ++i; GC_ASSERT(i < GC_n_memory); if ((word)GC_our_memory[i].hs_start<=(word)start){ start=GC_our_memory[i].hs_start +GC_our_memory[i].hs_bytes; ++i; } while (i < GC_n_memory &&(word)GC_our_memory[i].hs_start < (word)end &&(word)start < (word)end){ if ((word)start < (word)GC_our_memory[i].hs_start) GC_add_roots_inner(start, GC_our_memory[i].hs_start,TRUE); start=GC_our_memory[i].hs_start +GC_our_memory[i].hs_bytes; ++i; } if ((word)start < (word)end) GC_add_roots_inner(start,end,TRUE); } else if (prot[0]=='-'&&prot[1]=='-'&&prot[2]=='-'){ GC_remove_roots_subregion(start,end); } } } GC_INNER void GC_register_dynamic_libraries(void) { char*maps=GC_get_maps(); if (NULL==maps) ABORT("Failed to read/proc for library registration"); GC_register_map_entries(maps); } GC_INNER GC_bool GC_register_main_static_data(void) { return FALSE; } #define HAVE_REGISTER_MAIN_STATIC_DATA #else #if __GLIBC__ > 2||(__GLIBC__==2&&__GLIBC_MINOR__ > 2)||(__GLIBC__==2&&__GLIBC_MINOR__==2&&defined(DT_CONFIG))||defined(HOST_ANDROID) #ifndef HAVE_DL_ITERATE_PHDR #define HAVE_DL_ITERATE_PHDR #endif #ifdef HOST_ANDROID EXTERN_C_BEGIN extern int dl_iterate_phdr(int (*cb)(struct dl_phdr_info*, size_t,void*), void*data); EXTERN_C_END #endif #endif #if defined(__DragonFly__)||defined(__FreeBSD_kernel__)||(defined(FREEBSD)&&__FreeBSD__>=7) #ifndef HAVE_DL_ITERATE_PHDR #define HAVE_DL_ITERATE_PHDR #endif #define DL_ITERATE_PHDR_STRONG #elif defined(HAVE_DL_ITERATE_PHDR) EXTERN_C_BEGIN #pragma weak dl_iterate_phdr EXTERN_C_END #endif #if defined(HAVE_DL_ITERATE_PHDR) #ifdef PT_GNU_RELRO #define MAX_LOAD_SEGS MAX_ROOT_SETS static struct load_segment { ptr_t start; ptr_t end; ptr_t start2; ptr_t end2; } load_segs[MAX_LOAD_SEGS]; static int n_load_segs; static GC_bool load_segs_overflow; #endif STATIC int GC_register_dynlib_callback(struct dl_phdr_info*info, size_t size,void*ptr) { const ElfW(Phdr)*p; ptr_t start,end; int i; if (size < offsetof (struct dl_phdr_info,dlpi_phnum) +sizeof (info->dlpi_phnum)) return -1; p=info->dlpi_phdr; for (i=0;i < (int)info->dlpi_phnum;i++,p++){ if (p->p_type==PT_LOAD){ GC_has_static_roots_func callback=GC_has_static_roots; if ((p->p_flags&PF_W)==0)continue; start=(ptr_t)p->p_vaddr+info->dlpi_addr; end=start+p->p_memsz; if (callback!=0&&!callback(info->dlpi_name,start,p->p_memsz)) continue; #ifdef PT_GNU_RELRO #if CPP_WORDSZ==64 start=(ptr_t)((word)start&~(word)(sizeof(word)- 1)); #endif if (n_load_segs>=MAX_LOAD_SEGS){ if (!load_segs_overflow){ WARN("Too many PT_LOAD segments;" " registering as roots directly...\n",0); load_segs_overflow=TRUE; } GC_add_roots_inner(start,end,TRUE); } else { load_segs[n_load_segs].start=start; load_segs[n_load_segs].end=end; load_segs[n_load_segs].start2=0; load_segs[n_load_segs].end2=0; ++n_load_segs; } #else GC_add_roots_inner(start,end,TRUE); #endif } } #ifdef PT_GNU_RELRO p=info->dlpi_phdr; for (i=0;i < (int)info->dlpi_phnum;i++,p++){ if (p->p_type==PT_GNU_RELRO){ int j; start=(ptr_t)p->p_vaddr+info->dlpi_addr; end=start+p->p_memsz; for (j=n_load_segs;--j>=0;){ if ((word)start>=(word)load_segs[j].start &&(word)start < (word)load_segs[j].end){ if (load_segs[j].start2!=0){ WARN("More than one GNU_RELRO segment per load one\n",0); } else { GC_ASSERT((word)end<=(word)load_segs[j].end); load_segs[j].end2=load_segs[j].end; load_segs[j].end=start; load_segs[j].start2=end; } break; } if (0==j&&0==GC_has_static_roots) WARN("Failed to find PT_GNU_RELRO segment" " inside PT_LOAD region\n",0); } } } #endif *(int*)ptr=1; return 0; } GC_INNER GC_bool GC_register_main_static_data(void) { #ifdef DL_ITERATE_PHDR_STRONG return FALSE; #else return 0==COVERT_DATAFLOW(dl_iterate_phdr); #endif } STATIC GC_bool GC_register_dynamic_libraries_dl_iterate_phdr(void) { int did_something; if (GC_register_main_static_data()) return FALSE; #ifdef PT_GNU_RELRO { static GC_bool excluded_segs=FALSE; n_load_segs=0; load_segs_overflow=FALSE; if (!EXPECT(excluded_segs,TRUE)){ GC_exclude_static_roots_inner((ptr_t)load_segs, (ptr_t)load_segs+sizeof(load_segs)); excluded_segs=TRUE; } } #endif did_something=0; dl_iterate_phdr(GC_register_dynlib_callback,&did_something); if (did_something){ #ifdef PT_GNU_RELRO int i; for (i=0;i < n_load_segs;++i){ if ((word)load_segs[i].end > (word)load_segs[i].start){ GC_add_roots_inner(load_segs[i].start,load_segs[i].end,TRUE); } if ((word)load_segs[i].end2 > (word)load_segs[i].start2){ GC_add_roots_inner(load_segs[i].start2,load_segs[i].end2,TRUE); } } #endif } else { ptr_t datastart,dataend; #ifdef DATASTART_IS_FUNC static ptr_t datastart_cached=(ptr_t)GC_WORD_MAX; if (datastart_cached==(ptr_t)GC_WORD_MAX){ datastart_cached=DATASTART; } datastart=datastart_cached; #else datastart=DATASTART; #endif #ifdef DATAEND_IS_FUNC { static ptr_t dataend_cached=0; if (dataend_cached==0){ dataend_cached=DATAEND; } dataend=dataend_cached; } #else dataend=DATAEND; #endif if (NULL==*(char*volatile*)&datastart ||(word)datastart > (word)dataend) ABORT_ARG2("Wrong DATASTART/END pair", ":%p .. %p",(void*)datastart,(void*)dataend); GC_add_roots_inner(datastart,dataend,TRUE); #ifdef GC_HAVE_DATAREGION2 if ((word)DATASTART2 - 1U>=(word)DATAEND2){ ABORT_ARG2("Wrong DATASTART/END2 pair", ":%p .. %p",(void*)DATASTART2,(void*)DATAEND2); } GC_add_roots_inner(DATASTART2,DATAEND2,TRUE); #endif } return TRUE; } #define HAVE_REGISTER_MAIN_STATIC_DATA #else #if defined(NETBSD)||defined(OPENBSD) #include #ifndef DT_DEBUG #define DT_DEBUG 21 #endif #ifndef PT_LOAD #define PT_LOAD 1 #endif #ifndef PF_W #define PF_W 2 #endif #elif!defined(HOST_ANDROID) #include #endif #ifndef HOST_ANDROID #include #endif #endif EXTERN_C_BEGIN #ifdef __GNUC__ #pragma weak _DYNAMIC #endif extern ElfW(Dyn)_DYNAMIC[]; EXTERN_C_END STATIC struct link_map* GC_FirstDLOpenedLinkMap(void) { static struct link_map*cachedResult=0; if (0==COVERT_DATAFLOW(_DYNAMIC)){ return(0); } if( cachedResult==0){ #if defined(NETBSD)&&defined(RTLD_DI_LINKMAP) #if defined(CPPCHECK) #define GC_RTLD_DI_LINKMAP 2 #else #define GC_RTLD_DI_LINKMAP RTLD_DI_LINKMAP #endif struct link_map*lm=NULL; if (!dlinfo(RTLD_SELF,GC_RTLD_DI_LINKMAP,&lm)&&lm!=NULL){ while (lm->l_prev!=NULL){ lm=lm->l_prev; } cachedResult=lm->l_next; } #else ElfW(Dyn)*dp; int tag; for( dp=_DYNAMIC;(tag=dp->d_tag)!=0;dp++){ if (tag==DT_DEBUG){ struct r_debug*rd=(struct r_debug*)dp->d_un.d_ptr; if (rd!=NULL){ struct link_map*lm=rd->r_map; if (lm!=NULL) cachedResult=lm->l_next; } break; } } #endif } return cachedResult; } GC_INNER void GC_register_dynamic_libraries(void) { struct link_map*lm; #ifdef HAVE_DL_ITERATE_PHDR if (GC_register_dynamic_libraries_dl_iterate_phdr()){ return; } #endif for (lm=GC_FirstDLOpenedLinkMap();lm!=0;lm=lm->l_next) { ElfW(Ehdr)*e; ElfW(Phdr)*p; unsigned long offset; char*start; int i; e=(ElfW(Ehdr)*)lm->l_addr; #ifdef HOST_ANDROID if (e==NULL) continue; #endif p=((ElfW(Phdr)*)(((char*)(e))+e->e_phoff)); offset=((unsigned long)(lm->l_addr)); for( i=0;i < (int)e->e_phnum;i++,p++){ switch( p->p_type){ case PT_LOAD: { if(!(p->p_flags&PF_W))break; start=((char*)(p->p_vaddr))+offset; GC_add_roots_inner(start,start+p->p_memsz,TRUE); } break; default: break; } } } } #endif #endif #if defined(IRIX5)||(defined(USE_PROC_FOR_LIBRARIES)&&!defined(LINUX)) #include #include #include #include #include #include #ifndef _sigargs #define IRIX6 #endif GC_INNER void GC_register_dynamic_libraries(void) { static int fd=-1; char buf[30]; static prmap_t*addr_map=0; static int current_sz=0; int needed_sz=0; int i; long flags; ptr_t start; ptr_t limit; ptr_t heap_start=HEAP_START; ptr_t heap_end=heap_start; #ifdef SOLARISDL #define MA_PHYS 0 #endif if (fd < 0){ (void)snprintf(buf,sizeof(buf),"/proc/%ld",(long)getpid()); buf[sizeof(buf)- 1]='\0'; fd=open(buf,O_RDONLY); if (fd < 0){ ABORT("/proc open failed"); } } if (ioctl(fd,PIOCNMAP,&needed_sz)< 0){ ABORT_ARG2("/proc PIOCNMAP ioctl failed", ":fd=%d,errno=%d",fd,errno); } if (needed_sz>=current_sz){ GC_scratch_recycle_no_gww(addr_map, (size_t)current_sz*sizeof(prmap_t)); current_sz=needed_sz*2+1; addr_map=(prmap_t*)GC_scratch_alloc( (size_t)current_sz*sizeof(prmap_t)); if (addr_map==NULL) ABORT("Insufficient memory for address map"); } if (ioctl(fd,PIOCMAP,addr_map)< 0){ ABORT_ARG3("/proc PIOCMAP ioctl failed", ":errcode=%d,needed_sz=%d,addr_map=%p", errno,needed_sz,(void*)addr_map); }; if (GC_n_heap_sects > 0){ heap_end=GC_heap_sects[GC_n_heap_sects-1].hs_start +GC_heap_sects[GC_n_heap_sects-1].hs_bytes; if ((word)heap_end < (word)GC_scratch_last_end_ptr) heap_end=GC_scratch_last_end_ptr; } for (i=0;i < needed_sz;i++){ flags=addr_map[i].pr_mflags; if ((flags&(MA_BREAK|MA_STACK|MA_PHYS |MA_FETCHOP|MA_NOTCACHED))!=0)goto irrelevant; if ((flags&(MA_READ|MA_WRITE))!=(MA_READ|MA_WRITE)) goto irrelevant; start=(ptr_t)(addr_map[i].pr_vaddr); if (GC_roots_present(start))goto irrelevant; if ((word)start < (word)heap_end&&(word)start>=(word)heap_start) goto irrelevant; limit=start+addr_map[i].pr_size; #ifndef IRIX6 if (addr_map[i].pr_off==0&&strncmp(start,ELFMAG,4)==0){ caddr_t arg; int obj; #define MAP_IRR_SZ 10 static ptr_t map_irr[MAP_IRR_SZ]; static int n_irr=0; struct stat buf; int j; for (j=0;j < n_irr;j++){ if (map_irr[j]==start)goto irrelevant; } arg=(caddr_t)start; obj=ioctl(fd,PIOCOPENM,&arg); if (obj>=0){ fstat(obj,&buf); close(obj); if ((buf.st_mode&0111)!=0){ if (n_irr < MAP_IRR_SZ){ map_irr[n_irr++]=start; } goto irrelevant; } } } #endif GC_add_roots_inner(start,limit,TRUE); irrelevant:; } if (close(fd)< 0)ABORT("Couldn't close/proc file"); fd=-1; } #endif #if defined(MSWIN32)||defined(MSWINCE)||defined(CYGWIN32) #include STATIC void GC_cond_add_roots(char*base,char*limit) { #ifdef GC_WIN32_THREADS char*curr_base=base; char*next_stack_lo; char*next_stack_hi; if (base==limit)return; for(;;){ GC_get_next_stack(curr_base,limit,&next_stack_lo,&next_stack_hi); if ((word)next_stack_lo>=(word)limit)break; if ((word)next_stack_lo > (word)curr_base) GC_add_roots_inner(curr_base,next_stack_lo,TRUE); curr_base=next_stack_hi; } if ((word)curr_base < (word)limit) GC_add_roots_inner(curr_base,limit,TRUE); #else char*stack_top =(char*)((word)GC_approx_sp()& ~(word)(GC_sysinfo.dwAllocationGranularity - 1)); if (base==limit)return; if ((word)limit > (word)stack_top &&(word)base < (word)GC_stackbottom){ return; } GC_add_roots_inner(base,limit,TRUE); #endif } #ifdef DYNAMIC_LOADING GC_INNER GC_bool GC_register_main_static_data(void) { #if defined(MSWINCE)||defined(CYGWIN32) return FALSE; #else return GC_no_win32_dlls; #endif } #define HAVE_REGISTER_MAIN_STATIC_DATA #endif #ifdef DEBUG_VIRTUALQUERY void GC_dump_meminfo(MEMORY_BASIC_INFORMATION*buf) { GC_printf("BaseAddress=0x%lx,AllocationBase=0x%lx," " RegionSize=0x%lx(%lu)\n",buf->BaseAddress, buf->AllocationBase,buf->RegionSize,buf->RegionSize); GC_printf("\tAllocationProtect=0x%lx,State=0x%lx,Protect=0x%lx," "Type=0x%lx\n",buf->AllocationProtect,buf->State, buf->Protect,buf->Type); } #endif #if defined(MSWINCE)||defined(CYGWIN32) #define GC_wnt TRUE #endif GC_INNER void GC_register_dynamic_libraries(void) { MEMORY_BASIC_INFORMATION buf; DWORD protect; LPVOID p; char*base; char*limit,*new_limit; #ifdef MSWIN32 if (GC_no_win32_dlls)return; #endif p=GC_sysinfo.lpMinimumApplicationAddress; base=limit=(char*)p; while ((word)p < (word)GC_sysinfo.lpMaximumApplicationAddress){ size_t result=VirtualQuery(p,&buf,sizeof(buf)); #ifdef MSWINCE if (result==0){ new_limit=(char*) (((DWORD)p+GC_sysinfo.dwAllocationGranularity) &~(GC_sysinfo.dwAllocationGranularity-1)); } else #endif { if (result!=sizeof(buf)){ ABORT("Weird VirtualQuery result"); } new_limit=(char*)p+buf.RegionSize; protect=buf.Protect; if (buf.State==MEM_COMMIT &&(protect==PAGE_EXECUTE_READWRITE ||protect==PAGE_EXECUTE_WRITECOPY ||protect==PAGE_READWRITE ||protect==PAGE_WRITECOPY) &&(buf.Type==MEM_IMAGE #ifdef GC_REGISTER_MEM_PRIVATE ||(protect==PAGE_READWRITE&&buf.Type==MEM_PRIVATE) #else ||(!GC_wnt&&buf.Type==MEM_PRIVATE) #endif ) &&!GC_is_heap_base(buf.AllocationBase)){ #ifdef DEBUG_VIRTUALQUERY GC_dump_meminfo(&buf); #endif if ((char*)p!=limit){ GC_cond_add_roots(base,limit); base=(char*)p; } limit=new_limit; } } if ((word)p > (word)new_limit)break; p=(LPVOID)new_limit; } GC_cond_add_roots(base,limit); } #endif #if defined(ALPHA)&&defined(OSF1) #include EXTERN_C_BEGIN extern char*sys_errlist[]; extern int sys_nerr; extern int errno; EXTERN_C_END GC_INNER void GC_register_dynamic_libraries(void) { ldr_module_t moduleid=LDR_NULL_MODULE; ldr_process_t mypid=ldr_my_process(); while (TRUE){ ldr_module_info_t moduleinfo; size_t modulereturnsize; ldr_region_t region; ldr_region_info_t regioninfo; size_t regionreturnsize; int status=ldr_next_module(mypid,&moduleid); if (moduleid==LDR_NULL_MODULE) break; if (status!=0){ ABORT_ARG3("ldr_next_module failed", ":status=%d,errcode=%d (%s)",status,errno, errno < sys_nerr?sys_errlist[errno]:""); } status=ldr_inq_module(mypid,moduleid,&moduleinfo, sizeof(moduleinfo),&modulereturnsize); if (status!=0) ABORT("ldr_inq_module failed"); if (moduleinfo.lmi_flags&LDR_MAIN) continue; #ifdef DL_VERBOSE GC_log_printf("---Module---\n"); GC_log_printf("Module ID\t=%16ld\n",moduleinfo.lmi_modid); GC_log_printf("Count of regions=%16d\n",moduleinfo.lmi_nregion); GC_log_printf("flags for module=%16lx\n",moduleinfo.lmi_flags); GC_log_printf("module pathname\t=\"%s\"\n",moduleinfo.lmi_name); #endif for (region=0;region < moduleinfo.lmi_nregion;region++){ status=ldr_inq_region(mypid,moduleid,region,®ioninfo, sizeof(regioninfo),®ionreturnsize); if (status!=0) ABORT("ldr_inq_region failed"); if (!(regioninfo.lri_prot&LDR_W)) continue; #ifdef DL_VERBOSE GC_log_printf("--- Region---\n"); GC_log_printf("Region number\t=%16ld\n", regioninfo.lri_region_no); GC_log_printf("Protection flags=%016x\n",regioninfo.lri_prot); GC_log_printf("Virtual address\t=%16p\n",regioninfo.lri_vaddr); GC_log_printf("Mapped address\t=%16p\n", regioninfo.lri_mapaddr); GC_log_printf("Region size\t=%16ld\n",regioninfo.lri_size); GC_log_printf("Region name\t=\"%s\"\n",regioninfo.lri_name); #endif GC_add_roots_inner((char*)regioninfo.lri_mapaddr, (char*)regioninfo.lri_mapaddr+regioninfo.lri_size, TRUE); } } } #endif #if defined(HPUX) #include #include EXTERN_C_BEGIN extern char*sys_errlist[]; extern int sys_nerr; EXTERN_C_END GC_INNER void GC_register_dynamic_libraries(void) { int index=1; while (TRUE){ struct shl_descriptor*shl_desc; int status=shl_get(index,&shl_desc); if (status!=0){ #ifdef GC_HPUX_THREADS break; #else if (errno==EINVAL){ break; } else { ABORT_ARG3("shl_get failed", ":status=%d,errcode=%d (%s)",status,errno, errno < sys_nerr?sys_errlist[errno]:""); } #endif } #ifdef DL_VERBOSE GC_log_printf("---Shared library---\n"); GC_log_printf("\tfilename\t=\"%s\"\n",shl_desc->filename); GC_log_printf("\tindex\t\t=%d\n",index); GC_log_printf("\thandle\t\t=%08x\n", (unsigned long)shl_desc->handle); GC_log_printf("\ttext seg.start\t=%08x\n",shl_desc->tstart); GC_log_printf("\ttext seg.end\t=%08x\n",shl_desc->tend); GC_log_printf("\tdata seg.start\t=%08x\n",shl_desc->dstart); GC_log_printf("\tdata seg.end\t=%08x\n",shl_desc->dend); GC_log_printf("\tref.count\t=%lu\n",shl_desc->ref_count); #endif GC_add_roots_inner((char*)shl_desc->dstart, (char*)shl_desc->dend,TRUE); index++; } } #endif #ifdef AIX #include #include #include GC_INNER void GC_register_dynamic_libraries(void) { int ldibuflen=8192; for (;;){ int len; struct ld_info*ldi; #if defined(CPPCHECK) char ldibuf[ldibuflen]; #else char*ldibuf=alloca(ldibuflen); #endif len=loadquery(L_GETINFO,ldibuf,ldibuflen); if (len < 0){ if (errno!=ENOMEM){ ABORT("loadquery failed"); } ldibuflen*=2; continue; } ldi=(struct ld_info*)ldibuf; while (ldi){ len=ldi->ldinfo_next; GC_add_roots_inner( ldi->ldinfo_dataorg, (ptr_t)(unsigned long)ldi->ldinfo_dataorg +ldi->ldinfo_datasize, TRUE); ldi=len?(struct ld_info*)((char*)ldi+len):0; } break; } } #endif #ifdef DARWIN #ifndef __private_extern__ #define __private_extern__ extern #include #undef __private_extern__ #else #include #endif #include STATIC const struct dyld_sections_s { const char*seg; const char*sect; } GC_dyld_sections[]={ { SEG_DATA,SECT_DATA }, { SEG_DATA,"__static_data" }, { SEG_DATA,SECT_BSS }, { SEG_DATA,SECT_COMMON }, { SEG_DATA,"__zobj_data" }, { SEG_DATA,"__zobj_bss" } }; STATIC const char*const GC_dyld_add_sect_fmts[]={ "__bss%u", "__pu_bss%u", "__zo_bss%u", "__zo_pu_bss%u" }; #ifndef L2_MAX_OFILE_ALIGNMENT #define L2_MAX_OFILE_ALIGNMENT 15 #endif STATIC const char*GC_dyld_name_for_hdr(const struct GC_MACH_HEADER*hdr) { unsigned long i,c; c=_dyld_image_count(); for (i=0;i < c;i++) if ((const struct GC_MACH_HEADER*)_dyld_get_image_header(i)==hdr) return _dyld_get_image_name(i); return NULL; } STATIC void GC_dyld_image_add(const struct GC_MACH_HEADER*hdr, intptr_t slide) { unsigned long start,end; unsigned i,j; const struct GC_MACH_SECTION*sec; const char*name; GC_has_static_roots_func callback=GC_has_static_roots; DCL_LOCK_STATE; if (GC_no_dls)return; #ifdef DARWIN_DEBUG name=GC_dyld_name_for_hdr(hdr); #else name=callback!=0?GC_dyld_name_for_hdr(hdr):NULL; #endif for (i=0;i < sizeof(GC_dyld_sections)/sizeof(GC_dyld_sections[0]);i++){ sec=GC_GETSECTBYNAME(hdr,GC_dyld_sections[i].seg, GC_dyld_sections[i].sect); if (sec==NULL||sec->size < sizeof(word)) continue; start=slide+sec->addr; end=start+sec->size; LOCK(); if (callback==0||callback(name,(void*)start,(size_t)sec->size)){ #ifdef DARWIN_DEBUG GC_log_printf( "Adding section __DATA,%s at %p-%p (%lu bytes)from image %s\n", GC_dyld_sections[i].sect,(void*)start,(void*)end, (unsigned long)sec->size,name); #endif GC_add_roots_inner((ptr_t)start,(ptr_t)end,FALSE); } UNLOCK(); } for (j=0;j < sizeof(GC_dyld_add_sect_fmts)/sizeof(char*);j++){ const char*fmt=GC_dyld_add_sect_fmts[j]; for (i=0;i<=L2_MAX_OFILE_ALIGNMENT;i++){ char secnam[16]; (void)snprintf(secnam,sizeof(secnam),fmt,(unsigned)i); secnam[sizeof(secnam)- 1]='\0'; sec=GC_GETSECTBYNAME(hdr,SEG_DATA,secnam); if (sec==NULL||sec->size==0) continue; start=slide+sec->addr; end=start+sec->size; #ifdef DARWIN_DEBUG GC_log_printf("Adding on-demand section __DATA,%s at" " %p-%p (%lu bytes)from image %s\n", secnam,(void*)start,(void*)end, (unsigned long)sec->size,name); #endif GC_add_roots((char*)start,(char*)end); } } #if defined(DARWIN_DEBUG)&&!defined(NO_DEBUGGING) LOCK(); GC_print_static_roots(); UNLOCK(); #endif } STATIC void GC_dyld_image_remove(const struct GC_MACH_HEADER*hdr, intptr_t slide) { unsigned long start,end; unsigned i,j; const struct GC_MACH_SECTION*sec; #if defined(DARWIN_DEBUG)&&!defined(NO_DEBUGGING) DCL_LOCK_STATE; #endif for (i=0;i < sizeof(GC_dyld_sections)/sizeof(GC_dyld_sections[0]);i++){ sec=GC_GETSECTBYNAME(hdr,GC_dyld_sections[i].seg, GC_dyld_sections[i].sect); if (sec==NULL||sec->size==0) continue; start=slide+sec->addr; end=start+sec->size; #ifdef DARWIN_DEBUG GC_log_printf( "Removing section __DATA,%s at %p-%p (%lu bytes)from image %s\n", GC_dyld_sections[i].sect,(void*)start,(void*)end, (unsigned long)sec->size,GC_dyld_name_for_hdr(hdr)); #endif GC_remove_roots((char*)start,(char*)end); } for (j=0;j < sizeof(GC_dyld_add_sect_fmts)/sizeof(char*);j++){ const char*fmt=GC_dyld_add_sect_fmts[j]; for (i=0;i<=L2_MAX_OFILE_ALIGNMENT;i++){ char secnam[16]; (void)snprintf(secnam,sizeof(secnam),fmt,(unsigned)i); secnam[sizeof(secnam)- 1]='\0'; sec=GC_GETSECTBYNAME(hdr,SEG_DATA,secnam); if (sec==NULL||sec->size==0) continue; start=slide+sec->addr; end=start+sec->size; #ifdef DARWIN_DEBUG GC_log_printf("Removing on-demand section __DATA,%s at" " %p-%p (%lu bytes)from image %s\n",secnam, (void*)start,(void*)end,(unsigned long)sec->size, GC_dyld_name_for_hdr(hdr)); #endif GC_remove_roots((char*)start,(char*)end); } } #if defined(DARWIN_DEBUG)&&!defined(NO_DEBUGGING) LOCK(); GC_print_static_roots(); UNLOCK(); #endif } GC_INNER void GC_register_dynamic_libraries(void) { } GC_INNER void GC_init_dyld(void) { static GC_bool initialized=FALSE; if (initialized)return; #ifdef DARWIN_DEBUG GC_log_printf("Registering dyld callbacks...\n"); #endif _dyld_register_func_for_add_image( (void (*)(const struct mach_header*,intptr_t))GC_dyld_image_add); _dyld_register_func_for_remove_image( (void (*)(const struct mach_header*,intptr_t))GC_dyld_image_remove); initialized=TRUE; #ifdef NO_DYLD_BIND_FULLY_IMAGE #else if (GC_no_dls)return; if (GETENV("DYLD_BIND_AT_LAUNCH")==0){ #ifdef DARWIN_DEBUG GC_log_printf("Forcing full bind of GC code...\n"); #endif if (!_dyld_bind_fully_image_containing_address( (unsigned long*)GC_malloc)) ABORT("_dyld_bind_fully_image_containing_address failed"); } #endif } #define HAVE_REGISTER_MAIN_STATIC_DATA GC_INNER GC_bool GC_register_main_static_data(void) { return FALSE; } #endif #if defined(HAIKU) #include GC_INNER void GC_register_dynamic_libraries(void) { image_info info; int32 cookie=0; while (get_next_image_info(0,&cookie,&info)==B_OK){ ptr_t data=(ptr_t)info.data; GC_add_roots_inner(data,data+info.data_size,TRUE); } } #endif #elif defined(PCR) GC_INNER void GC_register_dynamic_libraries(void) { PCR_IL_LoadedFile*p=PCR_IL_GetLastLoadedFile(); PCR_IL_LoadedSegment*q; while (p!=NIL&&!(p->lf_commitPoint)){ p=p->lf_prev; } for (;p!=NIL;p=p->lf_prev){ for (q=p->lf_ls;q!=NIL;q=q->ls_next){ if ((q->ls_flags&PCR_IL_SegFlags_Traced_MASK) ==PCR_IL_SegFlags_Traced_on){ GC_add_roots_inner((ptr_t)q->ls_addr, (ptr_t)q->ls_addr+q->ls_bytes,TRUE); } } } } #endif #if!defined(HAVE_REGISTER_MAIN_STATIC_DATA)&&defined(DYNAMIC_LOADING) GC_INNER GC_bool GC_register_main_static_data(void) { return TRUE; } #endif GC_API void GC_CALL GC_register_has_static_roots_callback( GC_has_static_roots_func callback) { GC_has_static_roots=callback; } #if defined(GC_PTHREADS)&&!defined(GC_NO_DLOPEN) #undef GC_MUST_RESTORE_REDEFINED_DLOPEN #if defined(dlopen)&&!defined(GC_USE_LD_WRAP) #undef dlopen #define GC_MUST_RESTORE_REDEFINED_DLOPEN #endif #ifndef USE_PROC_FOR_LIBRARIES static void disable_gc_for_dlopen(void) { DCL_LOCK_STATE; LOCK(); while (GC_incremental&&GC_collection_in_progress()){ ENTER_GC(); GC_collect_a_little_inner(1000); EXIT_GC(); } ++GC_dont_gc; UNLOCK(); } #endif #ifdef GC_USE_LD_WRAP #define WRAP_DLFUNC(f)__wrap_##f #define REAL_DLFUNC(f)__real_##f void*REAL_DLFUNC(dlopen)(const char*,int); #else #define WRAP_DLFUNC(f)GC_##f #define REAL_DLFUNC(f)f #endif GC_API void*WRAP_DLFUNC(dlopen)(const char*path,int mode) { void*result; #ifndef USE_PROC_FOR_LIBRARIES disable_gc_for_dlopen(); #endif result=REAL_DLFUNC(dlopen)(path,mode); #ifndef USE_PROC_FOR_LIBRARIES GC_enable(); #endif return(result); } #ifdef GC_USE_LD_WRAP GC_API void*GC_dlopen(const char*path,int mode) { return dlopen(path,mode); } #endif #ifdef GC_MUST_RESTORE_REDEFINED_DLOPEN #define dlopen GC_dlopen #endif #endif #if!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PSP2) #include #ifdef AMIGA #ifndef __GNUC__ #include #else #include #endif #endif #if defined(MACOS)&&defined(__MWERKS__) #if defined(POWERPC) #define NONVOLATILE_GPR_COUNT 19 struct ppc_registers { unsigned long gprs[NONVOLATILE_GPR_COUNT]; }; typedef struct ppc_registers ppc_registers; #if defined(CPPCHECK) void getRegisters(ppc_registers*regs); #else asm static void getRegisters(register ppc_registers*regs) { stmw r13,regs->gprs blr } #endif static void PushMacRegisters(void) { ppc_registers regs; int i; getRegisters(®s); for (i=0;i < NONVOLATILE_GPR_COUNT;i++) GC_push_one(regs.gprs[i]); } #else asm static void PushMacRegisters(void) { sub.w #4,sp move.l a2,(sp) jsr GC_push_one move.l a3,(sp) jsr GC_push_one move.l a4,(sp) jsr GC_push_one #if!__option(a6frames) move.l a6,(sp) jsr GC_push_one #endif move.l d2,(sp) jsr GC_push_one move.l d3,(sp) jsr GC_push_one move.l d4,(sp) jsr GC_push_one move.l d5,(sp) jsr GC_push_one move.l d6,(sp) jsr GC_push_one move.l d7,(sp) jsr GC_push_one add.w #4,sp rts } #endif #endif #if defined(SPARC)||defined(IA64) GC_INNER ptr_t GC_save_regs_ret_val=NULL; #endif #undef HAVE_PUSH_REGS #if defined(USE_ASM_PUSH_REGS) #define HAVE_PUSH_REGS #else #ifdef STACK_NOT_SCANNED void GC_push_regs(void) { } #define HAVE_PUSH_REGS #elif defined(M68K)&&defined(AMIGA) void GC_push_regs(void) { #ifdef __GNUC__ asm("subq.w&0x4,%sp"); asm("mov.l %a2,(%sp)");asm("jsr _GC_push_one"); asm("mov.l %a3,(%sp)");asm("jsr _GC_push_one"); asm("mov.l %a4,(%sp)");asm("jsr _GC_push_one"); asm("mov.l %a5,(%sp)");asm("jsr _GC_push_one"); asm("mov.l %a6,(%sp)");asm("jsr _GC_push_one"); asm("mov.l %d2,(%sp)");asm("jsr _GC_push_one"); asm("mov.l %d3,(%sp)");asm("jsr _GC_push_one"); asm("mov.l %d4,(%sp)");asm("jsr _GC_push_one"); asm("mov.l %d5,(%sp)");asm("jsr _GC_push_one"); asm("mov.l %d6,(%sp)");asm("jsr _GC_push_one"); asm("mov.l %d7,(%sp)");asm("jsr _GC_push_one"); asm("addq.w&0x4,%sp"); #else GC_push_one(getreg(REG_A2)); GC_push_one(getreg(REG_A3)); #ifndef __SASC GC_push_one(getreg(REG_A4)); #endif GC_push_one(getreg(REG_A5)); GC_push_one(getreg(REG_A6)); GC_push_one(getreg(REG_D2)); GC_push_one(getreg(REG_D3)); GC_push_one(getreg(REG_D4)); GC_push_one(getreg(REG_D5)); GC_push_one(getreg(REG_D6)); GC_push_one(getreg(REG_D7)); #endif } #define HAVE_PUSH_REGS #elif defined(MACOS) #if defined(M68K)&&defined(THINK_C)&&!defined(CPPCHECK) #define PushMacReg(reg)move.l reg,(sp)jsr GC_push_one void GC_push_regs(void) { asm { sub.w #4,sp;reserve space for one parameter. PushMacReg(a2); PushMacReg(a3); PushMacReg(a4); ;skip a5 (globals),a6 (frame pointer),and a7 (stack pointer) PushMacReg(d2); PushMacReg(d3); PushMacReg(d4); PushMacReg(d5); PushMacReg(d6); PushMacReg(d7); add.w #4,sp;fix stack. } } #define HAVE_PUSH_REGS #undef PushMacReg #elif defined(__MWERKS__) void GC_push_regs(void) { PushMacRegisters(); } #define HAVE_PUSH_REGS #endif #endif #endif #if defined(HAVE_PUSH_REGS)&&defined(THREADS) #error GC_push_regs cannot be used with threads #undef HAVE_PUSH_REGS #endif #if!defined(HAVE_PUSH_REGS)&&defined(UNIX_LIKE) #include #ifndef NO_GETCONTEXT #if defined(DARWIN)&&(MAC_OS_X_VERSION_MAX_ALLOWED>=1060) #include #else #include #endif #ifdef GETCONTEXT_FPU_EXCMASK_BUG #include #endif #endif #endif GC_ATTR_NO_SANITIZE_ADDR GC_INNER void GC_with_callee_saves_pushed(void (*fn)(ptr_t,void*), volatile ptr_t arg) { volatile int dummy; volatile ptr_t context=0; #if defined(HAVE_PUSH_REGS) GC_push_regs(); #elif defined(__EMSCRIPTEN__) #else #if defined(UNIX_LIKE)&&!defined(NO_GETCONTEXT) static signed char getcontext_works=0; ucontext_t ctxt; #ifdef GETCONTEXT_FPU_EXCMASK_BUG #ifdef X86_64 unsigned short old_fcw; #if defined(CPPCHECK) GC_noop1((word)&old_fcw); #endif __asm__ __volatile__ ("fstcw %0":"=m" (*&old_fcw)); #else int except_mask=fegetexcept(); #endif #endif if (getcontext_works>=0){ if (getcontext(&ctxt)< 0){ WARN("getcontext failed:" " using another register retrieval method...\n",0); } else { context=(ptr_t)&ctxt; } if (EXPECT(0==getcontext_works,FALSE)) getcontext_works=context!=NULL?1:-1; } #ifdef GETCONTEXT_FPU_EXCMASK_BUG #ifdef X86_64 __asm__ __volatile__ ("fldcw %0"::"m" (*&old_fcw)); { unsigned mxcsr; __asm__ __volatile__ ("stmxcsr %0":"=m" (*&mxcsr)); mxcsr=(mxcsr&~(FE_ALL_EXCEPT<<7))| ((old_fcw&FE_ALL_EXCEPT)<<7); __asm__ __volatile__ ("ldmxcsr %0"::"m" (*&mxcsr)); } #else if (feenableexcept(except_mask)< 0) ABORT("feenableexcept failed"); #endif #endif #if defined(SPARC)||defined(IA64) GC_save_regs_ret_val=GC_save_regs_in_stack(); #endif if (NULL==context) #endif { #if defined(HAVE_BUILTIN_UNWIND_INIT) __builtin_unwind_init(); #elif defined(NO_CRT)&&defined(MSWIN32) CONTEXT ctx; RtlCaptureContext(&ctx); #else jmp_buf regs; word*i=(word*)®s; ptr_t lim=(ptr_t)(®s)+sizeof(regs); for (;(word)i < (word)lim;i++){ *i=0; } #if defined(MSWIN32)||defined(MSWINCE)||defined(UTS4)||defined(OS2)||defined(CX_UX)||defined(__CC_ARM)||defined(LINUX)||defined(EWS4800)||defined(RTEMS) (void)setjmp(regs); #else (void)_setjmp(regs); #endif #endif } #endif fn(arg,( void*)context); GC_noop1(COVERT_DATAFLOW(&dummy)); } #endif #if defined(GC_PTHREADS)&&!defined(GC_WIN32_THREADS)&&!defined(GC_DARWIN_THREADS)&&!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PSP2) #ifdef NACL #include #include STATIC int GC_nacl_num_gc_threads=0; STATIC __thread int GC_nacl_thread_idx=-1; STATIC volatile int GC_nacl_park_threads_now=0; STATIC volatile pthread_t GC_nacl_thread_parker=-1; GC_INNER __thread GC_thread GC_nacl_gc_thread_self=NULL; volatile int GC_nacl_thread_parked[MAX_NACL_GC_THREADS]; int GC_nacl_thread_used[MAX_NACL_GC_THREADS]; #elif defined(GC_OPENBSD_UTHREADS) #include #else #include #include #include #include #include #if (!defined(AO_HAVE_load_acquire)||!defined(AO_HAVE_store_release))&&!defined(CPPCHECK) #error AO_load_acquire and/or AO_store_release are missing; #error please define AO_REQUIRE_CAS manually #endif #undef pthread_sigmask #ifdef GC_ENABLE_SUSPEND_THREAD static void*GC_CALLBACK suspend_self_inner(void*client_data); #endif #ifdef DEBUG_THREADS #ifndef NSIG #if defined(MAXSIG) #define NSIG (MAXSIG+1) #elif defined(_NSIG) #define NSIG _NSIG #elif defined(__SIGRTMAX) #define NSIG (__SIGRTMAX+1) #else #error define NSIG #endif #endif void GC_print_sig_mask(void) { sigset_t blocked; int i; if (pthread_sigmask(SIG_BLOCK,NULL,&blocked)!=0) ABORT("pthread_sigmask failed"); for (i=1;i < NSIG;i++){ if (sigismember(&blocked,i)) GC_printf("Signal blocked:%d\n",i); } } #endif STATIC void GC_remove_allowed_signals(sigset_t*set) { if (sigdelset(set,SIGINT)!=0 ||sigdelset(set,SIGQUIT)!=0 ||sigdelset(set,SIGABRT)!=0 ||sigdelset(set,SIGTERM)!=0){ ABORT("sigdelset failed"); } #ifdef MPROTECT_VDB if (sigdelset(set,SIGSEGV)!=0 #ifdef HAVE_SIGBUS ||sigdelset(set,SIGBUS)!=0 #endif ){ ABORT("sigdelset failed"); } #endif } static sigset_t suspend_handler_mask; #define THREAD_RESTARTED 0x1 STATIC volatile AO_t GC_stop_count=0; STATIC volatile AO_t GC_world_is_stopped=FALSE; #if defined(GC_OSF1_THREADS)||defined(THREAD_SANITIZER)||defined(ADDRESS_SANITIZER)||defined(MEMORY_SANITIZER) STATIC GC_bool GC_retry_signals=TRUE; #else STATIC GC_bool GC_retry_signals=FALSE; #endif #ifndef SIG_THR_RESTART #if defined(GC_HPUX_THREADS)||defined(GC_OSF1_THREADS)||defined(GC_NETBSD_THREADS)||defined(GC_USESIGRT_SIGNALS) #if defined(_SIGRTMIN)&&!defined(CPPCHECK) #define SIG_THR_RESTART _SIGRTMIN+5 #else #define SIG_THR_RESTART SIGRTMIN+5 #endif #else #define SIG_THR_RESTART SIGXCPU #endif #endif #define SIGNAL_UNSET (-1) STATIC int GC_sig_suspend=SIGNAL_UNSET; STATIC int GC_sig_thr_restart=SIGNAL_UNSET; GC_API void GC_CALL GC_set_suspend_signal(int sig) { if (GC_is_initialized)return; GC_sig_suspend=sig; } GC_API void GC_CALL GC_set_thr_restart_signal(int sig) { if (GC_is_initialized)return; GC_sig_thr_restart=sig; } GC_API int GC_CALL GC_get_suspend_signal(void) { return GC_sig_suspend!=SIGNAL_UNSET?GC_sig_suspend:SIG_SUSPEND; } GC_API int GC_CALL GC_get_thr_restart_signal(void) { return GC_sig_thr_restart!=SIGNAL_UNSET ?GC_sig_thr_restart:SIG_THR_RESTART; } #if defined(GC_EXPLICIT_SIGNALS_UNBLOCK)||!defined(NO_SIGNALS_UNBLOCK_IN_MAIN) GC_INNER void GC_unblock_gc_signals(void) { sigset_t set; sigemptyset(&set); GC_ASSERT(GC_sig_suspend!=SIGNAL_UNSET); GC_ASSERT(GC_sig_thr_restart!=SIGNAL_UNSET); sigaddset(&set,GC_sig_suspend); sigaddset(&set,GC_sig_thr_restart); if (pthread_sigmask(SIG_UNBLOCK,&set,NULL)!=0) ABORT("pthread_sigmask failed"); } #endif STATIC sem_t GC_suspend_ack_sem; STATIC void GC_suspend_handler_inner(ptr_t dummy,void*context); #ifndef NO_SA_SIGACTION STATIC void GC_suspend_handler(int sig,siginfo_t*info GC_ATTR_UNUSED, void*context GC_ATTR_UNUSED) #else STATIC void GC_suspend_handler(int sig) #endif { int old_errno=errno; if (sig!=GC_sig_suspend){ #if defined(GC_FREEBSD_THREADS) if (0==sig)return; #endif ABORT("Bad signal in suspend_handler"); } #if defined(IA64)||defined(HP_PA)||defined(M68K) GC_with_callee_saves_pushed(GC_suspend_handler_inner,NULL); #else { #ifdef NO_SA_SIGACTION void*context=0; #endif GC_suspend_handler_inner(NULL,context); } #endif errno=old_errno; } #ifdef BASE_ATOMIC_OPS_EMULATED #define ao_load_acquire_async(p)(*(p)) #define ao_load_async(p)ao_load_acquire_async(p) #define ao_store_release_async(p,v)(void)(*(p)=(v)) #define ao_store_async(p,v)ao_store_release_async(p,v) #else #define ao_load_acquire_async(p)AO_load_acquire(p) #define ao_load_async(p)AO_load(p) #define ao_store_release_async(p,v)AO_store_release(p,v) #define ao_store_async(p,v)AO_store(p,v) #endif #ifdef THREAD_SANITIZER GC_ATTR_NO_SANITIZE_THREAD static GC_thread GC_lookup_thread_async(pthread_t id) { GC_thread p=GC_threads[THREAD_TABLE_INDEX(id)]; while (p!=NULL&&!THREAD_EQUAL(p->id,id)) p=p->next; return p; } #else #define GC_lookup_thread_async GC_lookup_thread #endif GC_INLINE void GC_store_stack_ptr(GC_thread me) { #ifdef SPARC ao_store_async((volatile AO_t*)&me->stop_info.stack_ptr, (AO_t)GC_save_regs_in_stack()); #else #ifdef IA64 me->backing_store_ptr=GC_save_regs_in_stack(); #endif ao_store_async((volatile AO_t*)&me->stop_info.stack_ptr, (AO_t)GC_approx_sp()); #endif } STATIC void GC_suspend_handler_inner(ptr_t dummy GC_ATTR_UNUSED, void*context GC_ATTR_UNUSED) { pthread_t self=pthread_self(); GC_thread me; IF_CANCEL(int cancel_state;) AO_t my_stop_count=ao_load_acquire_async(&GC_stop_count); DISABLE_CANCEL(cancel_state); #ifdef DEBUG_THREADS GC_log_printf("Suspending %p\n",(void*)self); #endif GC_ASSERT(((word)my_stop_count&THREAD_RESTARTED)==0); me=GC_lookup_thread_async(self); #ifdef GC_ENABLE_SUSPEND_THREAD if (ao_load_async(&me->suspended_ext)){ GC_store_stack_ptr(me); sem_post(&GC_suspend_ack_sem); suspend_self_inner(me); #ifdef DEBUG_THREADS GC_log_printf("Continuing %p on GC_resume_thread\n",(void*)self); #endif RESTORE_CANCEL(cancel_state); return; } #endif if (((word)me->stop_info.last_stop_count&~(word)THREAD_RESTARTED) ==(word)my_stop_count){ if (!GC_retry_signals){ WARN("Duplicate suspend signal in thread %p\n",self); } RESTORE_CANCEL(cancel_state); return; } GC_store_stack_ptr(me); #ifdef THREAD_SANITIZER { sigset_t set; sigemptyset(&set); GC_ASSERT(GC_sig_suspend!=SIGNAL_UNSET); GC_ASSERT(GC_sig_thr_restart!=SIGNAL_UNSET); sigaddset(&set,GC_sig_suspend); sigaddset(&set,GC_sig_thr_restart); if (pthread_sigmask(SIG_UNBLOCK,&set,NULL)!=0) ABORT("pthread_sigmask failed in suspend handler"); } #endif sem_post(&GC_suspend_ack_sem); ao_store_release_async(&me->stop_info.last_stop_count,my_stop_count); do { sigsuspend (&suspend_handler_mask); } while (ao_load_acquire_async(&GC_world_is_stopped) &&ao_load_async(&GC_stop_count)==my_stop_count); #ifdef DEBUG_THREADS GC_log_printf("Continuing %p\n",(void*)self); #endif #ifndef GC_NETBSD_THREADS_WORKAROUND if (GC_retry_signals) #endif { sem_post(&GC_suspend_ack_sem); #ifdef GC_NETBSD_THREADS_WORKAROUND if (GC_retry_signals) #endif { ao_store_release_async(&me->stop_info.last_stop_count, (AO_t)((word)my_stop_count|THREAD_RESTARTED)); } } RESTORE_CANCEL(cancel_state); } static void suspend_restart_barrier(int n_live_threads) { int i; for (i=0;i < n_live_threads;i++){ while (0!=sem_wait(&GC_suspend_ack_sem)){ if (errno!=EINTR) ABORT("sem_wait failed"); } } #ifdef GC_ASSERTIONS sem_getvalue(&GC_suspend_ack_sem,&i); GC_ASSERT(0==i); #endif } static int resend_lost_signals(int n_live_threads, int (*suspend_restart_all)(void)) { #define WAIT_UNIT 3000 #define RETRY_INTERVAL 100000 if (n_live_threads > 0){ unsigned long wait_usecs=0; for (;;){ int ack_count; sem_getvalue(&GC_suspend_ack_sem,&ack_count); if (ack_count==n_live_threads) break; if (wait_usecs > RETRY_INTERVAL){ int newly_sent=suspend_restart_all(); GC_COND_LOG_PRINTF("Resent %d signals after timeout\n",newly_sent); sem_getvalue(&GC_suspend_ack_sem,&ack_count); if (newly_sent < n_live_threads - ack_count){ WARN("Lost some threads while stopping or starting world?!\n",0); n_live_threads=ack_count+newly_sent; } wait_usecs=0; } #ifdef LINT2 #undef WAIT_UNIT #define WAIT_UNIT 1 sched_yield(); #elif defined(CPPCHECK) { struct timespec ts; ts.tv_sec=0; ts.tv_nsec=WAIT_UNIT*1000; (void)nanosleep(&ts,NULL); } #else usleep(WAIT_UNIT); #endif wait_usecs+=WAIT_UNIT; } } return n_live_threads; } STATIC void GC_restart_handler(int sig) { #if defined(DEBUG_THREADS) int old_errno=errno; #endif if (sig!=GC_sig_thr_restart) ABORT("Bad signal in restart handler"); #ifdef DEBUG_THREADS GC_log_printf("In GC_restart_handler for %p\n",(void*)pthread_self()); errno=old_errno; #endif } #ifdef USE_TKILL_ON_ANDROID EXTERN_C_BEGIN extern int tkill(pid_t tid,int sig); EXTERN_C_END static int android_thread_kill(pid_t tid,int sig) { int ret; int old_errno=errno; ret=tkill(tid,sig); if (ret < 0){ ret=errno; errno=old_errno; } return ret; } #define THREAD_SYSTEM_ID(t)(t)->kernel_id #define RAISE_SIGNAL(t,sig)android_thread_kill(THREAD_SYSTEM_ID(t),sig) #else #define THREAD_SYSTEM_ID(t)(t)->id #define RAISE_SIGNAL(t,sig)pthread_kill(THREAD_SYSTEM_ID(t),sig) #endif #ifdef GC_ENABLE_SUSPEND_THREAD #include STATIC void GC_brief_async_signal_safe_sleep(void) { struct timeval tv; tv.tv_sec=0; #if defined(GC_TIME_LIMIT)&&!defined(CPPCHECK) tv.tv_usec=1000*GC_TIME_LIMIT/2; #else tv.tv_usec=1000*50/2; #endif (void)select(0,0,0,0,&tv); } static void*GC_CALLBACK suspend_self_inner(void*client_data){ GC_thread me=(GC_thread)client_data; while (ao_load_acquire_async(&me->suspended_ext)){ GC_brief_async_signal_safe_sleep(); } return NULL; } GC_API void GC_CALL GC_suspend_thread(GC_SUSPEND_THREAD_ID thread){ GC_thread t; IF_CANCEL(int cancel_state;) DCL_LOCK_STATE; LOCK(); t=GC_lookup_thread((pthread_t)thread); if (t==NULL||t->suspended_ext){ UNLOCK(); return; } AO_store_release(&t->suspended_ext,TRUE); if (THREAD_EQUAL((pthread_t)thread,pthread_self())){ UNLOCK(); (void)GC_do_blocking(suspend_self_inner,t); return; } if ((t->flags&FINISHED)!=0){ UNLOCK(); return; } DISABLE_CANCEL(cancel_state); #ifdef PARALLEL_MARK if (GC_parallel) GC_wait_for_reclaim(); #endif if (GC_manual_vdb){ GC_acquire_dirty_lock(); } switch (RAISE_SIGNAL(t,GC_sig_suspend)){ case 0: break; default: ABORT("pthread_kill failed"); } GC_ASSERT(GC_thr_initialized); while (sem_wait(&GC_suspend_ack_sem)!=0){ if (errno!=EINTR) ABORT("sem_wait for handler failed (suspend_self)"); } if (GC_manual_vdb) GC_release_dirty_lock(); RESTORE_CANCEL(cancel_state); UNLOCK(); } GC_API void GC_CALL GC_resume_thread(GC_SUSPEND_THREAD_ID thread){ GC_thread t; DCL_LOCK_STATE; LOCK(); t=GC_lookup_thread((pthread_t)thread); if (t!=NULL) AO_store(&t->suspended_ext,FALSE); UNLOCK(); } GC_API int GC_CALL GC_is_thread_suspended(GC_SUSPEND_THREAD_ID thread){ GC_thread t; int is_suspended=0; DCL_LOCK_STATE; LOCK(); t=GC_lookup_thread((pthread_t)thread); if (t!=NULL&&t->suspended_ext) is_suspended=(int)TRUE; UNLOCK(); return is_suspended; } #endif #undef ao_load_acquire_async #undef ao_load_async #undef ao_store_async #undef ao_store_release_async #endif #ifdef IA64 #define IF_IA64(x)x #else #define IF_IA64(x) #endif GC_INNER void GC_push_all_stacks(void) { GC_bool found_me=FALSE; size_t nthreads=0; int i; GC_thread p; ptr_t lo,hi; IF_IA64(ptr_t bs_lo;ptr_t bs_hi;) struct GC_traced_stack_sect_s*traced_stack_sect; pthread_t self=pthread_self(); word total_size=0; if (!EXPECT(GC_thr_initialized,TRUE)) GC_thr_init(); #ifdef DEBUG_THREADS GC_log_printf("Pushing stacks from thread %p\n",(void*)self); #endif for (i=0;i < THREAD_TABLE_SZ;i++){ for (p=GC_threads[i];p!=0;p=p->next){ if (p->flags&FINISHED)continue; ++nthreads; traced_stack_sect=p->traced_stack_sect; if (THREAD_EQUAL(p->id,self)){ GC_ASSERT(!p->thread_blocked); #ifdef SPARC lo=(ptr_t)GC_save_regs_in_stack(); #else lo=GC_approx_sp(); #endif found_me=TRUE; IF_IA64(bs_hi=(ptr_t)GC_save_regs_in_stack();) } else { lo=(ptr_t)AO_load((volatile AO_t*)&p->stop_info.stack_ptr); IF_IA64(bs_hi=p->backing_store_ptr;) if (traced_stack_sect!=NULL &&traced_stack_sect->saved_stack_ptr==lo){ traced_stack_sect=traced_stack_sect->prev; } } if ((p->flags&MAIN_THREAD)==0){ hi=p->stack_end; IF_IA64(bs_lo=p->backing_store_end); } else { hi=GC_stackbottom; IF_IA64(bs_lo=BACKING_STORE_BASE;) } #ifdef DEBUG_THREADS GC_log_printf("Stack for thread %p=[%p,%p)\n", (void*)p->id,(void*)lo,(void*)hi); #endif if (0==lo)ABORT("GC_push_all_stacks:sp not set!"); if (p->altstack!=NULL&&(word)p->altstack<=(word)lo &&(word)lo<=(word)p->altstack+p->altstack_size){ hi=p->altstack+p->altstack_size; } GC_push_all_stack_sections(lo,hi,traced_stack_sect); #ifdef STACK_GROWS_UP total_size+=lo - hi; #else total_size+=hi - lo; #endif #ifdef NACL GC_push_all_stack((ptr_t)p->stop_info.reg_storage, (ptr_t)(p->stop_info.reg_storage+NACL_GC_REG_STORAGE_SIZE)); total_size+=NACL_GC_REG_STORAGE_SIZE*sizeof(ptr_t); #endif #ifdef IA64 #ifdef DEBUG_THREADS GC_log_printf("Reg stack for thread %p=[%p,%p)\n", (void*)p->id,(void*)bs_lo,(void*)bs_hi); #endif GC_push_all_register_sections(bs_lo,bs_hi, THREAD_EQUAL(p->id,self), traced_stack_sect); total_size+=bs_hi - bs_lo; #endif } } GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks\n",(int)nthreads); if (!found_me&&!GC_in_thread_creation) ABORT("Collecting from unknown thread"); GC_total_stacksize=total_size; } #ifdef DEBUG_THREADS pthread_t GC_stopping_thread; int GC_stopping_pid=0; #endif STATIC int GC_suspend_all(void) { int n_live_threads=0; int i; #ifndef NACL GC_thread p; #ifndef GC_OPENBSD_UTHREADS int result; #endif pthread_t self=pthread_self(); for (i=0;i < THREAD_TABLE_SZ;i++){ for (p=GC_threads[i];p!=0;p=p->next){ if (!THREAD_EQUAL(p->id,self)){ if ((p->flags&FINISHED)!=0)continue; if (p->thread_blocked)continue; #ifndef GC_OPENBSD_UTHREADS #ifdef GC_ENABLE_SUSPEND_THREAD if (p->suspended_ext)continue; #endif if (AO_load(&p->stop_info.last_stop_count)==GC_stop_count) continue; n_live_threads++; #endif #ifdef DEBUG_THREADS GC_log_printf("Sending suspend signal to %p\n",(void*)p->id); #endif #ifdef GC_OPENBSD_UTHREADS { stack_t stack; GC_acquire_dirty_lock(); if (pthread_suspend_np(p->id)!=0) ABORT("pthread_suspend_np failed"); GC_release_dirty_lock(); if (pthread_stackseg_np(p->id,&stack)) ABORT("pthread_stackseg_np failed"); p->stop_info.stack_ptr=(ptr_t)stack.ss_sp - stack.ss_size; if (GC_on_thread_event) GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, (void*)p->id); } #else result=RAISE_SIGNAL(p,GC_sig_suspend); switch(result){ case ESRCH: n_live_threads--; break; case 0: if (GC_on_thread_event) GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, (void*)(word)THREAD_SYSTEM_ID(p)); break; default: ABORT_ARG1("pthread_kill failed at suspend", ":errcode=%d",result); } #endif } } } #else #ifndef NACL_PARK_WAIT_NANOSECONDS #define NACL_PARK_WAIT_NANOSECONDS (100*1000) #endif #define NANOS_PER_SECOND (1000UL*1000*1000) unsigned long num_sleeps=0; #ifdef DEBUG_THREADS GC_log_printf("pthread_stop_world:num_threads=%d\n", GC_nacl_num_gc_threads - 1); #endif GC_nacl_thread_parker=pthread_self(); GC_nacl_park_threads_now=1; if (GC_manual_vdb) GC_acquire_dirty_lock(); while (1){ int num_threads_parked=0; struct timespec ts; int num_used=0; for (i=0;i < MAX_NACL_GC_THREADS &&num_used < GC_nacl_num_gc_threads;i++){ if (GC_nacl_thread_used[i]==1){ num_used++; if (GC_nacl_thread_parked[i]==1){ num_threads_parked++; if (GC_on_thread_event) GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,(void*)(word)i); } } } if (num_threads_parked>=GC_nacl_num_gc_threads - 1) break; ts.tv_sec=0; ts.tv_nsec=NACL_PARK_WAIT_NANOSECONDS; #ifdef DEBUG_THREADS GC_log_printf("Sleep waiting for %d threads to park...\n", GC_nacl_num_gc_threads - num_threads_parked - 1); #endif nanosleep(&ts,0); if (++num_sleeps > NANOS_PER_SECOND/NACL_PARK_WAIT_NANOSECONDS){ WARN("GC appears stalled waiting for %" WARN_PRIdPTR " threads to park...\n", GC_nacl_num_gc_threads - num_threads_parked - 1); num_sleeps=0; } } if (GC_manual_vdb) GC_release_dirty_lock(); #endif return n_live_threads; } GC_INNER void GC_stop_world(void) { #if!defined(GC_OPENBSD_UTHREADS)&&!defined(NACL) int n_live_threads; #endif GC_ASSERT(I_HOLD_LOCK()); #ifdef DEBUG_THREADS GC_stopping_thread=pthread_self(); GC_stopping_pid=getpid(); GC_log_printf("Stopping the world from %p\n",(void*)GC_stopping_thread); #endif #ifdef PARALLEL_MARK if (GC_parallel){ GC_acquire_mark_lock(); GC_ASSERT(GC_fl_builder_count==0); } #endif #if defined(GC_OPENBSD_UTHREADS)||defined(NACL) (void)GC_suspend_all(); #else AO_store(&GC_stop_count, (AO_t)((word)GC_stop_count+(THREAD_RESTARTED+1))); if (GC_manual_vdb){ GC_acquire_dirty_lock(); } AO_store_release(&GC_world_is_stopped,TRUE); n_live_threads=GC_suspend_all(); if (GC_retry_signals) n_live_threads=resend_lost_signals(n_live_threads,GC_suspend_all); suspend_restart_barrier(n_live_threads); if (GC_manual_vdb) GC_release_dirty_lock(); #endif #ifdef PARALLEL_MARK if (GC_parallel) GC_release_mark_lock(); #endif #ifdef DEBUG_THREADS GC_log_printf("World stopped from %p\n",(void*)pthread_self()); GC_stopping_thread=0; #endif } #ifdef NACL #if defined(__x86_64__) #define NACL_STORE_REGS()do { __asm__ __volatile__ ("push %rbx");__asm__ __volatile__ ("push %rbp");__asm__ __volatile__ ("push %r12");__asm__ __volatile__ ("push %r13");__asm__ __volatile__ ("push %r14");__asm__ __volatile__ ("push %r15");__asm__ __volatile__ ("mov %%esp,%0":"=m" (GC_nacl_gc_thread_self->stop_info.stack_ptr));BCOPY(GC_nacl_gc_thread_self->stop_info.stack_ptr,GC_nacl_gc_thread_self->stop_info.reg_storage,NACL_GC_REG_STORAGE_SIZE*sizeof(ptr_t));__asm__ __volatile__ ("naclasp $48,%r15");} while (0) #elif defined(__i386__) #define NACL_STORE_REGS()do { __asm__ __volatile__ ("push %ebx");__asm__ __volatile__ ("push %ebp");__asm__ __volatile__ ("push %esi");__asm__ __volatile__ ("push %edi");__asm__ __volatile__ ("mov %%esp,%0":"=m" (GC_nacl_gc_thread_self->stop_info.stack_ptr));BCOPY(GC_nacl_gc_thread_self->stop_info.stack_ptr,GC_nacl_gc_thread_self->stop_info.reg_storage,NACL_GC_REG_STORAGE_SIZE*sizeof(ptr_t));__asm__ __volatile__ ("add $16,%esp");} while (0) #elif defined(__arm__) #define NACL_STORE_REGS()do { __asm__ __volatile__ ("push {r4-r8,r10-r12,lr}");__asm__ __volatile__ ("mov r0,%0"::"r" (&GC_nacl_gc_thread_self->stop_info.stack_ptr));__asm__ __volatile__ ("bic r0,r0,#0xc0000000");__asm__ __volatile__ ("str sp,[r0]");BCOPY(GC_nacl_gc_thread_self->stop_info.stack_ptr,GC_nacl_gc_thread_self->stop_info.reg_storage,NACL_GC_REG_STORAGE_SIZE*sizeof(ptr_t));__asm__ __volatile__ ("add sp,sp,#40");__asm__ __volatile__ ("bic sp,sp,#0xc0000000");} while (0) #else #error TODO Please port NACL_STORE_REGS #endif GC_API_OSCALL void nacl_pre_syscall_hook(void) { if (GC_nacl_thread_idx!=-1){ NACL_STORE_REGS(); GC_nacl_gc_thread_self->stop_info.stack_ptr=GC_approx_sp(); GC_nacl_thread_parked[GC_nacl_thread_idx]=1; } } GC_API_OSCALL void __nacl_suspend_thread_if_needed(void) { if (GC_nacl_park_threads_now){ pthread_t self=pthread_self(); if (GC_nacl_thread_parker==self) return; if (GC_nacl_thread_idx < 0) return; if (!GC_nacl_thread_parked[GC_nacl_thread_idx]){ NACL_STORE_REGS(); GC_nacl_gc_thread_self->stop_info.stack_ptr=GC_approx_sp(); } GC_nacl_thread_parked[GC_nacl_thread_idx]=1; while (GC_nacl_park_threads_now){ } GC_nacl_thread_parked[GC_nacl_thread_idx]=0; BZERO(GC_nacl_gc_thread_self->stop_info.reg_storage, NACL_GC_REG_STORAGE_SIZE*sizeof(ptr_t)); } } GC_API_OSCALL void nacl_post_syscall_hook(void) { __nacl_suspend_thread_if_needed(); if (GC_nacl_thread_idx!=-1){ GC_nacl_thread_parked[GC_nacl_thread_idx]=0; } } STATIC GC_bool GC_nacl_thread_parking_inited=FALSE; STATIC pthread_mutex_t GC_nacl_thread_alloc_lock=PTHREAD_MUTEX_INITIALIZER; struct nacl_irt_blockhook { int (*register_block_hooks)(void (*pre)(void),void (*post)(void)); }; EXTERN_C_BEGIN extern size_t nacl_interface_query(const char*interface_ident, void*table,size_t tablesize); EXTERN_C_END GC_INNER void GC_nacl_initialize_gc_thread(void) { int i; static struct nacl_irt_blockhook gc_hook; pthread_mutex_lock(&GC_nacl_thread_alloc_lock); if (!EXPECT(GC_nacl_thread_parking_inited,TRUE)){ BZERO(GC_nacl_thread_parked,sizeof(GC_nacl_thread_parked)); BZERO(GC_nacl_thread_used,sizeof(GC_nacl_thread_used)); nacl_interface_query("nacl-irt-blockhook-0.1", &gc_hook,sizeof(gc_hook)); gc_hook.register_block_hooks(nacl_pre_syscall_hook, nacl_post_syscall_hook); GC_nacl_thread_parking_inited=TRUE; } GC_ASSERT(GC_nacl_num_gc_threads<=MAX_NACL_GC_THREADS); for (i=0;i < MAX_NACL_GC_THREADS;i++){ if (GC_nacl_thread_used[i]==0){ GC_nacl_thread_used[i]=1; GC_nacl_thread_idx=i; GC_nacl_num_gc_threads++; break; } } pthread_mutex_unlock(&GC_nacl_thread_alloc_lock); } GC_INNER void GC_nacl_shutdown_gc_thread(void) { pthread_mutex_lock(&GC_nacl_thread_alloc_lock); GC_ASSERT(GC_nacl_thread_idx>=0); GC_ASSERT(GC_nacl_thread_idx < MAX_NACL_GC_THREADS); GC_ASSERT(GC_nacl_thread_used[GC_nacl_thread_idx]!=0); GC_nacl_thread_used[GC_nacl_thread_idx]=0; GC_nacl_thread_idx=-1; GC_nacl_num_gc_threads--; pthread_mutex_unlock(&GC_nacl_thread_alloc_lock); } #else STATIC int GC_restart_all(void) { int n_live_threads=0; int i; pthread_t self=pthread_self(); GC_thread p; #ifndef GC_OPENBSD_UTHREADS int result; #endif for (i=0;i < THREAD_TABLE_SZ;i++){ for (p=GC_threads[i];p!=NULL;p=p->next){ if (!THREAD_EQUAL(p->id,self)){ if ((p->flags&FINISHED)!=0)continue; if (p->thread_blocked)continue; #ifndef GC_OPENBSD_UTHREADS #ifdef GC_ENABLE_SUSPEND_THREAD if (p->suspended_ext)continue; #endif if (GC_retry_signals &&AO_load(&p->stop_info.last_stop_count) ==(AO_t)((word)GC_stop_count|THREAD_RESTARTED)) continue; n_live_threads++; #endif #ifdef DEBUG_THREADS GC_log_printf("Sending restart signal to %p\n",(void*)p->id); #endif #ifdef GC_OPENBSD_UTHREADS if (pthread_resume_np(p->id)!=0) ABORT("pthread_resume_np failed"); if (GC_on_thread_event) GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED,(void*)p->id); #else result=RAISE_SIGNAL(p,GC_sig_thr_restart); switch(result){ case ESRCH: n_live_threads--; break; case 0: if (GC_on_thread_event) GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, (void*)(word)THREAD_SYSTEM_ID(p)); break; default: ABORT_ARG1("pthread_kill failed at resume", ":errcode=%d",result); } #endif } } } return n_live_threads; } #endif GC_INNER void GC_start_world(void) { #ifndef NACL int n_live_threads; GC_ASSERT(I_HOLD_LOCK()); #ifdef DEBUG_THREADS GC_log_printf("World starting\n"); #endif #ifndef GC_OPENBSD_UTHREADS AO_store_release(&GC_world_is_stopped,FALSE); #endif n_live_threads=GC_restart_all(); #ifdef GC_OPENBSD_UTHREADS (void)n_live_threads; #elif defined(GC_NETBSD_THREADS_WORKAROUND) if (GC_retry_signals) n_live_threads=resend_lost_signals(n_live_threads,GC_restart_all); suspend_restart_barrier(n_live_threads); #else if (GC_retry_signals){ n_live_threads=resend_lost_signals(n_live_threads,GC_restart_all); suspend_restart_barrier(n_live_threads); } #endif #ifdef DEBUG_THREADS GC_log_printf("World started\n"); #endif #else #ifdef DEBUG_THREADS GC_log_printf("World starting...\n"); #endif GC_nacl_park_threads_now=0; if (GC_on_thread_event) GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED,NULL); #endif } GC_INNER void GC_stop_init(void) { #if!defined(GC_OPENBSD_UTHREADS)&&!defined(NACL) struct sigaction act; char*str; if (SIGNAL_UNSET==GC_sig_suspend) GC_sig_suspend=SIG_SUSPEND; if (SIGNAL_UNSET==GC_sig_thr_restart) GC_sig_thr_restart=SIG_THR_RESTART; if (GC_sig_suspend==GC_sig_thr_restart) ABORT("Cannot use same signal for thread suspend and resume"); if (sem_init(&GC_suspend_ack_sem,GC_SEM_INIT_PSHARED,0)!=0) ABORT("sem_init failed"); #ifdef SA_RESTART act.sa_flags=SA_RESTART #else act.sa_flags=0 #endif #ifndef NO_SA_SIGACTION |SA_SIGINFO #endif ; if (sigfillset(&act.sa_mask)!=0){ ABORT("sigfillset failed"); } #ifdef GC_RTEMS_PTHREADS if(sigprocmask(SIG_UNBLOCK,&act.sa_mask,NULL)!=0){ ABORT("sigprocmask failed"); } #endif GC_remove_allowed_signals(&act.sa_mask); #ifndef NO_SA_SIGACTION act.sa_sigaction=GC_suspend_handler; #else act.sa_handler=GC_suspend_handler; #endif if (sigaction(GC_sig_suspend,&act,NULL)!=0){ ABORT("Cannot set SIG_SUSPEND handler"); } #ifndef NO_SA_SIGACTION act.sa_flags&=~SA_SIGINFO; #endif act.sa_handler=GC_restart_handler; if (sigaction(GC_sig_thr_restart,&act,NULL)!=0){ ABORT("Cannot set SIG_THR_RESTART handler"); } if (sigfillset(&suspend_handler_mask)!=0)ABORT("sigfillset failed"); GC_remove_allowed_signals(&suspend_handler_mask); if (sigdelset(&suspend_handler_mask,GC_sig_thr_restart)!=0) ABORT("sigdelset failed"); str=GETENV("GC_RETRY_SIGNALS"); if (str!=NULL){ if (*str=='0'&&*(str+1)=='\0'){ GC_retry_signals=FALSE; } else { GC_retry_signals=TRUE; } } if (GC_retry_signals){ GC_COND_LOG_PRINTF( "Will retry suspend and restart signals if necessary\n"); } #ifndef NO_SIGNALS_UNBLOCK_IN_MAIN GC_unblock_gc_signals(); #endif #endif } #endif #if defined(GC_PTHREADS)&&!defined(GC_WIN32_THREADS) #include #include #include #include #include #include #if!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PSP2) #if!defined(GC_RTEMS_PTHREADS) #include #endif #include #include #include #include #endif #include #if defined(GC_DARWIN_THREADS) #ifndef GC_DARWIN_SEMAPHORE_H #define GC_DARWIN_SEMAPHORE_H #if!defined(GC_DARWIN_THREADS) #error darwin_semaphore.h included with GC_DARWIN_THREADS not defined #endif #ifdef __cplusplus extern "C" { #endif typedef struct { pthread_mutex_t mutex; pthread_cond_t cond; int value; } sem_t; GC_INLINE int sem_init(sem_t*sem,int pshared,int value){ if (pshared!=0){ errno=EPERM; return -1; } sem->value=value; if (pthread_mutex_init(&sem->mutex,NULL)!=0) return -1; if (pthread_cond_init(&sem->cond,NULL)!=0){ (void)pthread_mutex_destroy(&sem->mutex); return -1; } return 0; } GC_INLINE int sem_post(sem_t*sem){ if (pthread_mutex_lock(&sem->mutex)!=0) return -1; sem->value++; if (pthread_cond_signal(&sem->cond)!=0){ (void)pthread_mutex_unlock(&sem->mutex); return -1; } return pthread_mutex_unlock(&sem->mutex)!=0?-1:0; } GC_INLINE int sem_wait(sem_t*sem){ if (pthread_mutex_lock(&sem->mutex)!=0) return -1; while (sem->value==0){ if (pthread_cond_wait(&sem->cond,&sem->mutex)!=0){ (void)pthread_mutex_unlock(&sem->mutex); return -1; } } sem->value--; return pthread_mutex_unlock(&sem->mutex)!=0?-1:0; } GC_INLINE int sem_destroy(sem_t*sem){ return pthread_cond_destroy(&sem->cond)!=0 ||pthread_mutex_destroy(&sem->mutex)!=0?-1:0; } #ifdef __cplusplus } #endif #endif #else #include #endif #if defined(GC_DARWIN_THREADS)||defined(GC_FREEBSD_THREADS) #include #endif #if defined(GC_NETBSD_THREADS)||defined(GC_OPENBSD_THREADS) #include #include #endif #if!defined(USE_SPIN_LOCK) GC_INNER pthread_mutex_t GC_allocate_ml=PTHREAD_MUTEX_INITIALIZER; #endif #ifdef GC_ASSERTIONS GC_INNER unsigned long GC_lock_holder=NO_THREAD; #endif #if defined(GC_DGUX386_THREADS) #include #include typedef unsigned int sem_t; #endif #undef pthread_create #ifndef GC_NO_PTHREAD_SIGMASK #undef pthread_sigmask #endif #ifndef GC_NO_PTHREAD_CANCEL #undef pthread_cancel #endif #ifdef GC_HAVE_PTHREAD_EXIT #undef pthread_exit #endif #undef pthread_join #undef pthread_detach #if defined(GC_OSF1_THREADS)&&defined(_PTHREAD_USE_MANGLED_NAMES_)&&!defined(_PTHREAD_USE_PTDNAM_) #define pthread_create __pthread_create #define pthread_join __pthread_join #define pthread_detach __pthread_detach #ifndef GC_NO_PTHREAD_CANCEL #define pthread_cancel __pthread_cancel #endif #ifdef GC_HAVE_PTHREAD_EXIT #define pthread_exit __pthread_exit #endif #endif #ifdef GC_USE_LD_WRAP #define WRAP_FUNC(f)__wrap_##f #define REAL_FUNC(f)__real_##f int REAL_FUNC(pthread_create)(pthread_t*, GC_PTHREAD_CREATE_CONST pthread_attr_t*, void*(*start_routine)(void*),void*); int REAL_FUNC(pthread_join)(pthread_t,void**); int REAL_FUNC(pthread_detach)(pthread_t); #ifndef GC_NO_PTHREAD_SIGMASK int REAL_FUNC(pthread_sigmask)(int,const sigset_t*,sigset_t*); #endif #ifndef GC_NO_PTHREAD_CANCEL int REAL_FUNC(pthread_cancel)(pthread_t); #endif #ifdef GC_HAVE_PTHREAD_EXIT void REAL_FUNC(pthread_exit)(void*)GC_PTHREAD_EXIT_ATTRIBUTE; #endif #else #ifdef GC_USE_DLOPEN_WRAP #include #define WRAP_FUNC(f)f #define REAL_FUNC(f)GC_real_##f typedef int (*GC_pthread_create_t)(pthread_t*, GC_PTHREAD_CREATE_CONST pthread_attr_t*, void*(*)(void*),void*); static GC_pthread_create_t REAL_FUNC(pthread_create); #ifndef GC_NO_PTHREAD_SIGMASK typedef int (*GC_pthread_sigmask_t)(int,const sigset_t*, sigset_t*); static GC_pthread_sigmask_t REAL_FUNC(pthread_sigmask); #endif typedef int (*GC_pthread_join_t)(pthread_t,void**); static GC_pthread_join_t REAL_FUNC(pthread_join); typedef int (*GC_pthread_detach_t)(pthread_t); static GC_pthread_detach_t REAL_FUNC(pthread_detach); #ifndef GC_NO_PTHREAD_CANCEL typedef int (*GC_pthread_cancel_t)(pthread_t); static GC_pthread_cancel_t REAL_FUNC(pthread_cancel); #endif #ifdef GC_HAVE_PTHREAD_EXIT typedef void (*GC_pthread_exit_t)(void*)GC_PTHREAD_EXIT_ATTRIBUTE; static GC_pthread_exit_t REAL_FUNC(pthread_exit); #endif #else #define WRAP_FUNC(f)GC_##f #if!defined(GC_DGUX386_THREADS) #define REAL_FUNC(f)f #else #define REAL_FUNC(f)__d10_##f #endif #endif #endif #if defined(GC_USE_LD_WRAP)||defined(GC_USE_DLOPEN_WRAP) GC_API int GC_pthread_create(pthread_t*t, GC_PTHREAD_CREATE_CONST pthread_attr_t*a, void*(*fn)(void*),void*arg) { return pthread_create(t,a,fn,arg); } #ifndef GC_NO_PTHREAD_SIGMASK GC_API int GC_pthread_sigmask(int how,const sigset_t*mask, sigset_t*old) { return pthread_sigmask(how,mask,old); } #endif GC_API int GC_pthread_join(pthread_t t,void**res) { return pthread_join(t,res); } GC_API int GC_pthread_detach(pthread_t t) { return pthread_detach(t); } #ifndef GC_NO_PTHREAD_CANCEL GC_API int GC_pthread_cancel(pthread_t t) { return pthread_cancel(t); } #endif #ifdef GC_HAVE_PTHREAD_EXIT GC_API GC_PTHREAD_EXIT_ATTRIBUTE void GC_pthread_exit(void*retval) { pthread_exit(retval); } #endif #endif #ifdef GC_USE_DLOPEN_WRAP STATIC GC_bool GC_syms_initialized=FALSE; STATIC void GC_init_real_syms(void) { void*dl_handle; if (GC_syms_initialized)return; #ifdef RTLD_NEXT dl_handle=RTLD_NEXT; #else dl_handle=dlopen("libpthread.so.0",RTLD_LAZY); if (NULL==dl_handle){ dl_handle=dlopen("libpthread.so",RTLD_LAZY); } if (NULL==dl_handle)ABORT("Couldn't open libpthread"); #endif REAL_FUNC(pthread_create)=(GC_pthread_create_t)(word) dlsym(dl_handle,"pthread_create"); #ifdef RTLD_NEXT if (REAL_FUNC(pthread_create)==0) ABORT("pthread_create not found" " (probably -lgc is specified after -lpthread)"); #endif #ifndef GC_NO_PTHREAD_SIGMASK REAL_FUNC(pthread_sigmask)=(GC_pthread_sigmask_t)(word) dlsym(dl_handle,"pthread_sigmask"); #endif REAL_FUNC(pthread_join)=(GC_pthread_join_t)(word) dlsym(dl_handle,"pthread_join"); REAL_FUNC(pthread_detach)=(GC_pthread_detach_t)(word) dlsym(dl_handle,"pthread_detach"); #ifndef GC_NO_PTHREAD_CANCEL REAL_FUNC(pthread_cancel)=(GC_pthread_cancel_t)(word) dlsym(dl_handle,"pthread_cancel"); #endif #ifdef GC_HAVE_PTHREAD_EXIT REAL_FUNC(pthread_exit)=(GC_pthread_exit_t)(word) dlsym(dl_handle,"pthread_exit"); #endif GC_syms_initialized=TRUE; } #define INIT_REAL_SYMS()if (EXPECT(GC_syms_initialized,TRUE)){} else GC_init_real_syms() #define ASSERT_SYMS_INITIALIZED()GC_ASSERT(GC_syms_initialized) #else #define INIT_REAL_SYMS()(void)0 #define ASSERT_SYMS_INITIALIZED()GC_ASSERT(parallel_initialized) #endif static GC_bool parallel_initialized=FALSE; #ifndef GC_ALWAYS_MULTITHREADED GC_INNER GC_bool GC_need_to_lock=FALSE; #endif STATIC int GC_nprocs=1; #ifdef THREAD_LOCAL_ALLOC GC_INNER void GC_mark_thread_local_free_lists(void) { int i; GC_thread p; for (i=0;i < THREAD_TABLE_SZ;++i){ for (p=GC_threads[i];0!=p;p=p->next){ if (!(p->flags&FINISHED)) GC_mark_thread_local_fls_for(&(p->tlfs)); } } } #if defined(GC_ASSERTIONS) void GC_check_tls(void) { int i; GC_thread p; for (i=0;i < THREAD_TABLE_SZ;++i){ for (p=GC_threads[i];0!=p;p=p->next){ if (!(p->flags&FINISHED)) GC_check_tls_for(&(p->tlfs)); } } #if defined(USE_CUSTOM_SPECIFIC) if (GC_thread_key!=0) GC_check_tsd_marks(GC_thread_key); #endif } #endif #endif #ifdef PARALLEL_MARK #ifndef MAX_MARKERS #define MAX_MARKERS 16 #endif static ptr_t marker_sp[MAX_MARKERS - 1]={0}; #ifdef IA64 static ptr_t marker_bsp[MAX_MARKERS - 1]={0}; #endif #if defined(GC_DARWIN_THREADS)&&!defined(GC_NO_THREADS_DISCOVERY) static mach_port_t marker_mach_threads[MAX_MARKERS - 1]={0}; GC_INNER GC_bool GC_is_mach_marker(thread_act_t thread) { int i; for (i=0;i < GC_markers_m1;i++){ if (marker_mach_threads[i]==thread) return TRUE; } return FALSE; } #endif #ifdef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG static void set_marker_thread_name(unsigned id) { int err=pthread_setname_np(pthread_self(),"GC-marker-%zu", (void*)(size_t)id); if (err!=0) WARN("pthread_setname_np failed,errno=%" WARN_PRIdPTR "\n",err); } #elif defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID)||defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID) static void set_marker_thread_name(unsigned id) { char name_buf[16]; int len=sizeof("GC-marker-")- 1; BCOPY("GC-marker-",name_buf,len); if (id>=10) name_buf[len++]=(char)('0'+(id/10)% 10); name_buf[len]=(char)('0'+id % 10); name_buf[len+1]='\0'; #ifdef HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID (void)pthread_setname_np(name_buf); #else if (pthread_setname_np(pthread_self(),name_buf)!=0) WARN("pthread_setname_np failed\n",0); #endif } #else #define set_marker_thread_name(id)(void)(id) #endif STATIC void*GC_mark_thread(void*id) { word my_mark_no=0; IF_CANCEL(int cancel_state;) if ((word)id==GC_WORD_MAX)return 0; DISABLE_CANCEL(cancel_state); set_marker_thread_name((unsigned)(word)id); marker_sp[(word)id]=GC_approx_sp(); #ifdef IA64 marker_bsp[(word)id]=GC_save_regs_in_stack(); #endif #if defined(GC_DARWIN_THREADS)&&!defined(GC_NO_THREADS_DISCOVERY) marker_mach_threads[(word)id]=mach_thread_self(); #endif GC_acquire_mark_lock(); if (0==--GC_fl_builder_count) GC_notify_all_builder(); for (;;++my_mark_no){ if (my_mark_no < GC_mark_no||my_mark_no > GC_mark_no+2){ my_mark_no=GC_mark_no; } #ifdef DEBUG_THREADS GC_log_printf("Starting mark helper for mark number %lu\n", (unsigned long)my_mark_no); #endif GC_help_marker(my_mark_no); } } STATIC pthread_t GC_mark_threads[MAX_MARKERS]; #ifdef CAN_HANDLE_FORK static int available_markers_m1=0; static pthread_cond_t mark_cv; #else #define available_markers_m1 GC_markers_m1 static pthread_cond_t mark_cv=PTHREAD_COND_INITIALIZER; #endif GC_INNER void GC_start_mark_threads_inner(void) { int i; pthread_attr_t attr; #ifndef NO_MARKER_SPECIAL_SIGMASK sigset_t set,oldset; #endif GC_ASSERT(I_DONT_HOLD_LOCK()); if (available_markers_m1<=0)return; #ifdef CAN_HANDLE_FORK if (GC_parallel)return; { pthread_cond_t mark_cv_local=PTHREAD_COND_INITIALIZER; BCOPY(&mark_cv_local,&mark_cv,sizeof(mark_cv)); } #endif GC_ASSERT(GC_fl_builder_count==0); INIT_REAL_SYMS(); if (0!=pthread_attr_init(&attr))ABORT("pthread_attr_init failed"); if (0!=pthread_attr_setdetachstate(&attr,PTHREAD_CREATE_DETACHED)) ABORT("pthread_attr_setdetachstate failed"); #ifdef DEFAULT_STACK_MAYBE_SMALL { size_t old_size; if (pthread_attr_getstacksize(&attr,&old_size)!=0) ABORT("pthread_attr_getstacksize failed"); if (old_size < MIN_STACK_SIZE &&old_size!=0){ if (pthread_attr_setstacksize(&attr,MIN_STACK_SIZE)!=0) ABORT("pthread_attr_setstacksize failed"); } } #endif #ifndef NO_MARKER_SPECIAL_SIGMASK if (sigfillset(&set)!=0) ABORT("sigfillset failed"); #if!defined(GC_DARWIN_THREADS)&&!defined(GC_OPENBSD_UTHREADS)&&!defined(NACL) if (sigdelset(&set,GC_get_suspend_signal())!=0 ||sigdelset(&set,GC_get_thr_restart_signal())!=0) ABORT("sigdelset failed"); #endif if (REAL_FUNC(pthread_sigmask)(SIG_BLOCK,&set,&oldset)< 0){ WARN("pthread_sigmask set failed,no markers started," " errno=%" WARN_PRIdPTR "\n",errno); GC_markers_m1=0; (void)pthread_attr_destroy(&attr); return; } #endif #ifdef CAN_HANDLE_FORK GC_markers_m1=available_markers_m1; #endif for (i=0;i < available_markers_m1;++i){ if (0!=REAL_FUNC(pthread_create)(GC_mark_threads+i,&attr, GC_mark_thread,(void*)(word)i)){ WARN("Marker thread creation failed,errno=%" WARN_PRIdPTR "\n", errno); GC_markers_m1=i; break; } } #ifndef NO_MARKER_SPECIAL_SIGMASK if (REAL_FUNC(pthread_sigmask)(SIG_SETMASK,&oldset,NULL)< 0){ WARN("pthread_sigmask restore failed,errno=%" WARN_PRIdPTR "\n", errno); } #endif (void)pthread_attr_destroy(&attr); GC_wait_for_markers_init(); GC_COND_LOG_PRINTF("Started %d mark helper threads\n",GC_markers_m1); } #endif GC_INNER GC_bool GC_thr_initialized=FALSE; GC_INNER volatile GC_thread GC_threads[THREAD_TABLE_SZ]={0}; void GC_push_thread_structures(void) { GC_ASSERT(I_HOLD_LOCK()); GC_PUSH_ALL_SYM(GC_threads); #if defined(THREAD_LOCAL_ALLOC) GC_PUSH_ALL_SYM(GC_thread_key); #endif } #ifdef DEBUG_THREADS STATIC int GC_count_threads(void) { int i; int count=0; GC_ASSERT(I_HOLD_LOCK()); for (i=0;i < THREAD_TABLE_SZ;++i){ GC_thread th=GC_threads[i]; while (th){ if (!(th->flags&FINISHED)) ++count; th=th->next; } } return count; } #endif static struct GC_Thread_Rep first_thread; STATIC GC_thread GC_new_thread(pthread_t id) { int hv=THREAD_TABLE_INDEX(id); GC_thread result; static GC_bool first_thread_used=FALSE; #ifdef DEBUG_THREADS GC_log_printf("Creating thread %p\n",(void*)id); for (result=GC_threads[hv];result!=NULL;result=result->next) if (!THREAD_EQUAL(result->id,id)){ GC_log_printf("Hash collision at GC_threads[%d]\n",hv); break; } #endif GC_ASSERT(I_HOLD_LOCK()); if (!EXPECT(first_thread_used,TRUE)){ result=&first_thread; first_thread_used=TRUE; GC_ASSERT(NULL==GC_threads[hv]); #if defined(THREAD_SANITIZER)&&defined(CPPCHECK) GC_noop1(result->dummy[0]); #endif } else { result=(struct GC_Thread_Rep*) GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep),NORMAL); if (result==0)return(0); } result->id=id; #ifdef USE_TKILL_ON_ANDROID result->kernel_id=gettid(); #endif result->next=GC_threads[hv]; GC_threads[hv]=result; #ifdef NACL GC_nacl_gc_thread_self=result; GC_nacl_initialize_gc_thread(); #endif GC_ASSERT(result->flags==0&&result->thread_blocked==0); if (EXPECT(result!=&first_thread,TRUE)) GC_dirty(result); return(result); } STATIC void GC_delete_thread(pthread_t id) { int hv=THREAD_TABLE_INDEX(id); GC_thread p=GC_threads[hv]; GC_thread prev=NULL; #ifdef DEBUG_THREADS GC_log_printf("Deleting thread %p,n_threads=%d\n", (void*)id,GC_count_threads()); #endif #ifdef NACL GC_nacl_shutdown_gc_thread(); GC_nacl_gc_thread_self=NULL; #endif GC_ASSERT(I_HOLD_LOCK()); while (!THREAD_EQUAL(p->id,id)){ prev=p; p=p->next; } if (prev==0){ GC_threads[hv]=p->next; } else { GC_ASSERT(prev!=&first_thread); prev->next=p->next; GC_dirty(prev); } if (p!=&first_thread){ #ifdef GC_DARWIN_THREADS mach_port_deallocate(mach_task_self(),p->stop_info.mach_thread); #endif GC_INTERNAL_FREE(p); } } STATIC void GC_delete_gc_thread(GC_thread t) { pthread_t id=t->id; int hv=THREAD_TABLE_INDEX(id); GC_thread p=GC_threads[hv]; GC_thread prev=NULL; GC_ASSERT(I_HOLD_LOCK()); while (p!=t){ prev=p; p=p->next; } if (prev==0){ GC_threads[hv]=p->next; } else { GC_ASSERT(prev!=&first_thread); prev->next=p->next; GC_dirty(prev); } #ifdef GC_DARWIN_THREADS mach_port_deallocate(mach_task_self(),p->stop_info.mach_thread); #endif GC_INTERNAL_FREE(p); #ifdef DEBUG_THREADS GC_log_printf("Deleted thread %p,n_threads=%d\n", (void*)id,GC_count_threads()); #endif } GC_INNER GC_thread GC_lookup_thread(pthread_t id) { GC_thread p=GC_threads[THREAD_TABLE_INDEX(id)]; while (p!=0&&!THREAD_EQUAL(p->id,id))p=p->next; return(p); } GC_INNER void GC_reset_finalizer_nested(void) { GC_thread me=GC_lookup_thread(pthread_self()); me->finalizer_nested=0; } GC_INNER unsigned char*GC_check_finalizer_nested(void) { GC_thread me=GC_lookup_thread(pthread_self()); unsigned nesting_level=me->finalizer_nested; if (nesting_level){ if (++me->finalizer_skipped < (1U<finalizer_skipped=0; } me->finalizer_nested=(unsigned char)(nesting_level+1); return&me->finalizer_nested; } #if defined(GC_ASSERTIONS)&&defined(THREAD_LOCAL_ALLOC) GC_bool GC_is_thread_tsd_valid(void*tsd) { GC_thread me; DCL_LOCK_STATE; LOCK(); me=GC_lookup_thread(pthread_self()); UNLOCK(); return (word)tsd>=(word)(&me->tlfs) &&(word)tsd < (word)(&me->tlfs)+sizeof(me->tlfs); } #endif GC_API int GC_CALL GC_thread_is_registered(void) { pthread_t self=pthread_self(); GC_thread me; DCL_LOCK_STATE; LOCK(); me=GC_lookup_thread(self); UNLOCK(); return me!=NULL; } static pthread_t main_pthread_id; static void*main_stack,*main_altstack; static word main_stack_size,main_altstack_size; GC_API void GC_CALL GC_register_altstack(void*stack,GC_word stack_size, void*altstack, GC_word altstack_size) { GC_thread me; pthread_t self=pthread_self(); DCL_LOCK_STATE; LOCK(); me=GC_lookup_thread(self); if (me!=NULL){ me->stack=(ptr_t)stack; me->stack_size=stack_size; me->altstack=(ptr_t)altstack; me->altstack_size=altstack_size; } else { main_pthread_id=self; main_stack=stack; main_stack_size=stack_size; main_altstack=altstack; main_altstack_size=altstack_size; } UNLOCK(); } #ifdef CAN_HANDLE_FORK #ifdef CAN_CALL_ATFORK GC_ATTR_NO_SANITIZE_THREAD #endif static void store_to_threads_table(int hv,GC_thread me) { GC_threads[hv]=me; } STATIC void GC_remove_all_threads_but_me(void) { pthread_t self=pthread_self(); int hv; for (hv=0;hv < THREAD_TABLE_SZ;++hv){ GC_thread p,next; GC_thread me=NULL; for (p=GC_threads[hv];0!=p;p=next){ next=p->next; if (THREAD_EQUAL(p->id,self) &&me==NULL){ me=p; p->next=0; #ifdef GC_DARWIN_THREADS me->stop_info.mach_thread=mach_thread_self(); #endif #ifdef USE_TKILL_ON_ANDROID me->kernel_id=gettid(); #endif #if defined(THREAD_LOCAL_ALLOC)&&!defined(USE_CUSTOM_SPECIFIC) { int res; res=GC_setspecific(GC_thread_key,&me->tlfs); if (COVERT_DATAFLOW(res)!=0) ABORT("GC_setspecific failed (in child)"); } #endif } else { #ifdef THREAD_LOCAL_ALLOC if (!(p->flags&FINISHED)){ GC_remove_specific_after_fork(GC_thread_key,p->id); } #endif #if!defined(THREAD_SANITIZER)||!defined(CAN_CALL_ATFORK) if (p!=&first_thread)GC_INTERNAL_FREE(p); #endif } } store_to_threads_table(hv,me); } } #endif #ifdef USE_PROC_FOR_LIBRARIES GC_INNER GC_bool GC_segment_is_thread_stack(ptr_t lo,ptr_t hi) { int i; GC_thread p; GC_ASSERT(I_HOLD_LOCK()); #ifdef PARALLEL_MARK for (i=0;i < GC_markers_m1;++i){ if ((word)marker_sp[i] > (word)lo&&(word)marker_sp[i] < (word)hi) return TRUE; #ifdef IA64 if ((word)marker_bsp[i] > (word)lo &&(word)marker_bsp[i] < (word)hi) return TRUE; #endif } #endif for (i=0;i < THREAD_TABLE_SZ;i++){ for (p=GC_threads[i];p!=0;p=p->next){ if (0!=p->stack_end){ #ifdef STACK_GROWS_UP if ((word)p->stack_end>=(word)lo &&(word)p->stack_end < (word)hi) return TRUE; #else if ((word)p->stack_end > (word)lo &&(word)p->stack_end<=(word)hi) return TRUE; #endif } } } return FALSE; } #endif #ifdef IA64 GC_INNER ptr_t GC_greatest_stack_base_below(ptr_t bound) { int i; GC_thread p; ptr_t result=0; GC_ASSERT(I_HOLD_LOCK()); #ifdef PARALLEL_MARK for (i=0;i < GC_markers_m1;++i){ if ((word)marker_sp[i] > (word)result &&(word)marker_sp[i] < (word)bound) result=marker_sp[i]; } #endif for (i=0;i < THREAD_TABLE_SZ;i++){ for (p=GC_threads[i];p!=0;p=p->next){ if ((word)p->stack_end > (word)result &&(word)p->stack_end < (word)bound){ result=p->stack_end; } } } return result; } #endif #ifndef STAT_READ #define STAT_BUF_SIZE 4096 #define STAT_READ read #endif #ifdef GC_HPUX_THREADS #define GC_get_nprocs()pthread_num_processors_np() #elif defined(GC_OSF1_THREADS)||defined(GC_AIX_THREADS)||defined(GC_HAIKU_THREADS)||defined(GC_SOLARIS_THREADS)||defined(HURD)||defined(HOST_ANDROID)||defined(NACL) GC_INLINE int GC_get_nprocs(void) { int nprocs=(int)sysconf(_SC_NPROCESSORS_ONLN); return nprocs > 0?nprocs:1; } #elif defined(GC_IRIX_THREADS) GC_INLINE int GC_get_nprocs(void) { int nprocs=(int)sysconf(_SC_NPROC_ONLN); return nprocs > 0?nprocs:1; } #elif defined(GC_LINUX_THREADS) STATIC int GC_get_nprocs(void) { char stat_buf[STAT_BUF_SIZE]; int f; int result,i,len; f=open("/proc/stat",O_RDONLY); if (f < 0){ WARN("Couldn't read/proc/stat\n",0); return 1; } len=STAT_READ(f,stat_buf,STAT_BUF_SIZE); close(f); result=1; for (i=0;i < len - 100;++i){ if (stat_buf[i]=='\n'&&stat_buf[i+1]=='c' &&stat_buf[i+2]=='p'&&stat_buf[i+3]=='u'){ int cpu_no=atoi(&stat_buf[i+4]); if (cpu_no>=result) result=cpu_no+1; } } return result; } #elif defined(GC_DGUX386_THREADS) STATIC int GC_get_nprocs(void) { int numCpus; struct dg_sys_info_pm_info pm_sysinfo; int status=0; status=dg_sys_info((long int*)&pm_sysinfo, DG_SYS_INFO_PM_INFO_TYPE,DG_SYS_INFO_PM_CURRENT_VERSION); if (status < 0) numCpus=-1; else numCpus=pm_sysinfo.idle_vp_count; return(numCpus); } #elif defined(GC_DARWIN_THREADS)||defined(GC_FREEBSD_THREADS)||defined(GC_NETBSD_THREADS)||defined(GC_OPENBSD_THREADS) STATIC int GC_get_nprocs(void) { int mib[]={CTL_HW,HW_NCPU}; int res; size_t len=sizeof(res); sysctl(mib,sizeof(mib)/sizeof(int),&res,&len,NULL,0); return res; } #else #define GC_get_nprocs()1 #endif #if defined(ARM32)&&defined(GC_LINUX_THREADS)&&!defined(NACL) STATIC int GC_get_nprocs_present(void) { char stat_buf[16]; int f; int len; f=open("/sys/devices/system/cpu/present",O_RDONLY); if (f < 0) return -1; len=STAT_READ(f,stat_buf,sizeof(stat_buf)); close(f); if (len < 2||stat_buf[0]!='0'||stat_buf[len - 1]!='\n'){ return 0; } else if (len==2){ return 1; } else if (stat_buf[1]!='-'){ return 0; } stat_buf[len - 1]='\0'; return atoi(&stat_buf[2])+1; } #endif STATIC void GC_wait_for_gc_completion(GC_bool wait_for_all) { DCL_LOCK_STATE; #if!defined(THREAD_SANITIZER)||!defined(CAN_CALL_ATFORK) GC_ASSERT(I_HOLD_LOCK()); #endif ASSERT_CANCEL_DISABLED(); if (GC_incremental&&GC_collection_in_progress()){ word old_gc_no=GC_gc_no; while (GC_incremental&&GC_collection_in_progress() &&(wait_for_all||old_gc_no==GC_gc_no)){ ENTER_GC(); GC_in_thread_creation=TRUE; GC_collect_a_little_inner(1); GC_in_thread_creation=FALSE; EXIT_GC(); UNLOCK(); sched_yield(); LOCK(); } } } #ifdef CAN_HANDLE_FORK IF_CANCEL(static int fork_cancel_state;) #if defined(GC_ASSERTIONS)&&defined(CAN_CALL_ATFORK) GC_ATTR_NO_SANITIZE_THREAD #endif static void fork_prepare_proc(void) { LOCK(); DISABLE_CANCEL(fork_cancel_state); #if defined(PARALLEL_MARK) if (GC_parallel) GC_wait_for_reclaim(); #endif GC_wait_for_gc_completion(TRUE); #if defined(PARALLEL_MARK) if (GC_parallel) GC_acquire_mark_lock(); #endif GC_acquire_dirty_lock(); } #if defined(GC_ASSERTIONS)&&defined(CAN_CALL_ATFORK) GC_ATTR_NO_SANITIZE_THREAD #endif static void fork_parent_proc(void) { GC_release_dirty_lock(); #if defined(PARALLEL_MARK) if (GC_parallel) GC_release_mark_lock(); #endif RESTORE_CANCEL(fork_cancel_state); UNLOCK(); } #if defined(GC_ASSERTIONS)&&defined(CAN_CALL_ATFORK) GC_ATTR_NO_SANITIZE_THREAD #endif static void fork_child_proc(void) { GC_release_dirty_lock(); #if defined(PARALLEL_MARK) if (GC_parallel) GC_release_mark_lock(); #endif GC_remove_all_threads_but_me(); #ifdef PARALLEL_MARK GC_parallel=FALSE; #endif RESTORE_CANCEL(fork_cancel_state); UNLOCK(); #ifdef USE_PTHREAD_LOCKS GC_ASSERT(I_DONT_HOLD_LOCK()); (void)pthread_mutex_destroy(&GC_allocate_ml); if (0!=pthread_mutex_init(&GC_allocate_ml,NULL)) ABORT("pthread_mutex_init failed (in child)"); #endif } GC_API void GC_CALL GC_atfork_prepare(void) { if (!EXPECT(GC_is_initialized,TRUE))GC_init(); #if defined(GC_DARWIN_THREADS)&&defined(MPROTECT_VDB) if (GC_auto_incremental){ GC_ASSERT(0==GC_handle_fork); ABORT("Unable to fork while mprotect_thread is running"); } #endif if (GC_handle_fork<=0) fork_prepare_proc(); } GC_API void GC_CALL GC_atfork_parent(void) { if (GC_handle_fork<=0) fork_parent_proc(); } GC_API void GC_CALL GC_atfork_child(void) { if (GC_handle_fork<=0) fork_child_proc(); } #endif #ifdef INCLUDE_LINUX_THREAD_DESCR __thread int GC_dummy_thread_local; #endif #ifdef PARALLEL_MARK static void setup_mark_lock(void); static unsigned required_markers_cnt=0; #endif GC_API void GC_CALL GC_set_markers_count(unsigned markers GC_ATTR_UNUSED) { #ifdef PARALLEL_MARK required_markers_cnt=markers < MAX_MARKERS?markers:MAX_MARKERS; #endif } GC_INNER void GC_thr_init(void) { GC_ASSERT(I_HOLD_LOCK()); if (GC_thr_initialized)return; GC_thr_initialized=TRUE; GC_ASSERT((word)&GC_threads % sizeof(word)==0); #ifdef CAN_HANDLE_FORK if (GC_handle_fork){ #ifdef CAN_CALL_ATFORK if (pthread_atfork(fork_prepare_proc,fork_parent_proc, fork_child_proc)==0){ GC_handle_fork=1; } else #endif if (GC_handle_fork!=-1) ABORT("pthread_atfork failed"); } #endif #ifdef INCLUDE_LINUX_THREAD_DESCR { ptr_t thread_local_addr=(ptr_t)(&GC_dummy_thread_local); ptr_t main_thread_start,main_thread_end; if (!GC_enclosing_mapping(thread_local_addr,&main_thread_start, &main_thread_end)){ ABORT("Failed to find mapping for main thread thread locals"); } else { GC_add_roots_inner(main_thread_start,main_thread_end,FALSE); } } #endif { pthread_t self=pthread_self(); GC_thread t=GC_new_thread(self); if (t==NULL) ABORT("Failed to allocate memory for the initial thread"); #ifdef GC_DARWIN_THREADS t->stop_info.mach_thread=mach_thread_self(); #else t->stop_info.stack_ptr=GC_approx_sp(); #endif t->flags=DETACHED|MAIN_THREAD; if (THREAD_EQUAL(self,main_pthread_id)){ t->stack=(ptr_t)main_stack; t->stack_size=main_stack_size; t->altstack=(ptr_t)main_altstack; t->altstack_size=main_altstack_size; } } #ifndef GC_DARWIN_THREADS GC_stop_init(); #endif { char*nprocs_string=GETENV("GC_NPROCS"); GC_nprocs=-1; if (nprocs_string!=NULL)GC_nprocs=atoi(nprocs_string); } if (GC_nprocs<=0 #if defined(ARM32)&&defined(GC_LINUX_THREADS)&&!defined(NACL) &&(GC_nprocs=GC_get_nprocs_present())<=1 #endif ) { GC_nprocs=GC_get_nprocs(); } if (GC_nprocs<=0){ WARN("GC_get_nprocs()returned %" WARN_PRIdPTR "\n",GC_nprocs); GC_nprocs=2; #ifdef PARALLEL_MARK available_markers_m1=0; #endif } else { #ifdef PARALLEL_MARK { char*markers_string=GETENV("GC_MARKERS"); int markers=required_markers_cnt; if (markers_string!=NULL){ markers=atoi(markers_string); if (markers<=0||markers > MAX_MARKERS){ WARN("Too big or invalid number of mark threads:%" WARN_PRIdPTR ";using maximum threads\n",(signed_word)markers); markers=MAX_MARKERS; } } else if (0==markers){ markers=GC_nprocs; #if defined(GC_MIN_MARKERS)&&!defined(CPPCHECK) if (markers < GC_MIN_MARKERS) markers=GC_MIN_MARKERS; #endif if (markers > MAX_MARKERS) markers=MAX_MARKERS; } available_markers_m1=markers - 1; } #endif } GC_COND_LOG_PRINTF("Number of processors=%d\n",GC_nprocs); #ifdef PARALLEL_MARK if (available_markers_m1<=0){ GC_parallel=FALSE; GC_COND_LOG_PRINTF( "Single marker thread,turning off parallel marking\n"); } else { setup_mark_lock(); } #endif } GC_INNER void GC_init_parallel(void) { #if defined(THREAD_LOCAL_ALLOC) DCL_LOCK_STATE; #endif if (parallel_initialized)return; parallel_initialized=TRUE; if (!GC_is_initialized)GC_init(); #if defined(THREAD_LOCAL_ALLOC) LOCK(); GC_init_thread_local(&(GC_lookup_thread(pthread_self())->tlfs)); UNLOCK(); #endif } #ifndef GC_NO_PTHREAD_SIGMASK GC_API int WRAP_FUNC(pthread_sigmask)(int how,const sigset_t*set, sigset_t*oset) { sigset_t fudged_set; INIT_REAL_SYMS(); if (set!=NULL&&(how==SIG_BLOCK||how==SIG_SETMASK)){ int sig_suspend=GC_get_suspend_signal(); fudged_set=*set; GC_ASSERT(sig_suspend>=0); if (sigdelset(&fudged_set,sig_suspend)!=0) ABORT("sigdelset failed"); set=&fudged_set; } return(REAL_FUNC(pthread_sigmask)(how,set,oset)); } #endif GC_INNER void GC_do_blocking_inner(ptr_t data,void*context GC_ATTR_UNUSED) { struct blocking_data*d=(struct blocking_data*)data; pthread_t self=pthread_self(); GC_thread me; #if defined(SPARC)||defined(IA64) ptr_t stack_ptr=GC_save_regs_in_stack(); #endif #if defined(GC_DARWIN_THREADS)&&!defined(DARWIN_DONT_PARSE_STACK) GC_bool topOfStackUnset=FALSE; #endif DCL_LOCK_STATE; LOCK(); me=GC_lookup_thread(self); GC_ASSERT(!(me->thread_blocked)); #ifdef SPARC me->stop_info.stack_ptr=stack_ptr; #else me->stop_info.stack_ptr=GC_approx_sp(); #endif #if defined(GC_DARWIN_THREADS)&&!defined(DARWIN_DONT_PARSE_STACK) if (me->topOfStack==NULL){ topOfStackUnset=TRUE; me->topOfStack=GC_FindTopOfStack(0); } #endif #ifdef IA64 me->backing_store_ptr=stack_ptr; #endif me->thread_blocked=(unsigned char)TRUE; UNLOCK(); d->client_data=(d->fn)(d->client_data); LOCK(); #if defined(CPPCHECK) GC_noop1((word)&me->thread_blocked); #endif me->thread_blocked=FALSE; #if defined(GC_DARWIN_THREADS)&&!defined(DARWIN_DONT_PARSE_STACK) if (topOfStackUnset) me->topOfStack=NULL; #endif UNLOCK(); } GC_API void GC_CALL GC_set_stackbottom(void*gc_thread_handle, const struct GC_stack_base*sb) { GC_thread t=(GC_thread)gc_thread_handle; GC_ASSERT(sb->mem_base!=NULL); if (!EXPECT(GC_is_initialized,TRUE)){ GC_ASSERT(NULL==t); } else { GC_ASSERT(I_HOLD_LOCK()); if (NULL==t) t=GC_lookup_thread(pthread_self()); GC_ASSERT((t->flags&FINISHED)==0); GC_ASSERT(!(t->thread_blocked) &&NULL==t->traced_stack_sect); if ((t->flags&MAIN_THREAD)==0){ t->stack_end=(ptr_t)sb->mem_base; #ifdef IA64 t->backing_store_end=(ptr_t)sb->reg_base; #endif return; } } GC_stackbottom=(char*)sb->mem_base; #ifdef IA64 GC_register_stackbottom=(ptr_t)sb->reg_base; #endif } GC_API void*GC_CALL GC_get_my_stackbottom(struct GC_stack_base*sb) { pthread_t self=pthread_self(); GC_thread me; DCL_LOCK_STATE; LOCK(); me=GC_lookup_thread(self); if ((me->flags&MAIN_THREAD)==0){ sb->mem_base=me->stack_end; #ifdef IA64 sb->reg_base=me->backing_store_end; #endif } else { sb->mem_base=GC_stackbottom; #ifdef IA64 sb->reg_base=GC_register_stackbottom; #endif } UNLOCK(); return (void*)me; } GC_API void*GC_CALL GC_call_with_gc_active(GC_fn_type fn, void*client_data) { struct GC_traced_stack_sect_s stacksect; pthread_t self=pthread_self(); GC_thread me; DCL_LOCK_STATE; LOCK(); me=GC_lookup_thread(self); if ((me->flags&MAIN_THREAD)==0){ GC_ASSERT(me->stack_end!=NULL); if ((word)me->stack_end HOTTER_THAN (word)(&stacksect)) me->stack_end=(ptr_t)(&stacksect); } else { if ((word)GC_stackbottom HOTTER_THAN (word)(&stacksect)) GC_stackbottom=(ptr_t)COVERT_DATAFLOW(&stacksect); } if (!me->thread_blocked){ UNLOCK(); client_data=fn(client_data); GC_noop1(COVERT_DATAFLOW(&stacksect)); return client_data; } stacksect.saved_stack_ptr=me->stop_info.stack_ptr; #ifdef IA64 stacksect.backing_store_end=GC_save_regs_in_stack(); stacksect.saved_backing_store_ptr=me->backing_store_ptr; #endif stacksect.prev=me->traced_stack_sect; me->thread_blocked=FALSE; me->traced_stack_sect=&stacksect; UNLOCK(); client_data=fn(client_data); GC_ASSERT(me->thread_blocked==FALSE); GC_ASSERT(me->traced_stack_sect==&stacksect); #if defined(CPPCHECK) GC_noop1((word)me->traced_stack_sect); #endif LOCK(); me->traced_stack_sect=stacksect.prev; #ifdef IA64 me->backing_store_ptr=stacksect.saved_backing_store_ptr; #endif me->thread_blocked=(unsigned char)TRUE; me->stop_info.stack_ptr=stacksect.saved_stack_ptr; UNLOCK(); return client_data; } STATIC void GC_unregister_my_thread_inner(GC_thread me) { #ifdef DEBUG_THREADS GC_log_printf( "Unregistering thread %p,gc_thread=%p,n_threads=%d\n", (void*)me->id,(void*)me,GC_count_threads()); #endif GC_ASSERT(!(me->flags&FINISHED)); #if defined(THREAD_LOCAL_ALLOC) GC_ASSERT(GC_getspecific(GC_thread_key)==&me->tlfs); GC_destroy_thread_local(&(me->tlfs)); #endif #if defined(GC_HAVE_PTHREAD_EXIT)||!defined(GC_NO_PTHREAD_CANCEL) if ((me->flags&DISABLED_GC)!=0){ GC_dont_gc--; } #endif if (me->flags&DETACHED){ GC_delete_thread(pthread_self()); } else { me->flags|=FINISHED; } #if defined(THREAD_LOCAL_ALLOC) GC_remove_specific(GC_thread_key); #endif } GC_API int GC_CALL GC_unregister_my_thread(void) { pthread_t self=pthread_self(); GC_thread me; IF_CANCEL(int cancel_state;) DCL_LOCK_STATE; LOCK(); DISABLE_CANCEL(cancel_state); GC_wait_for_gc_completion(FALSE); me=GC_lookup_thread(self); #ifdef DEBUG_THREADS GC_log_printf( "Called GC_unregister_my_thread on %p,gc_thread=%p\n", (void*)self,(void*)me); #endif GC_ASSERT(THREAD_EQUAL(me->id,self)); GC_unregister_my_thread_inner(me); RESTORE_CANCEL(cancel_state); UNLOCK(); return GC_SUCCESS; } GC_INNER_PTHRSTART void GC_thread_exit_proc(void*arg) { IF_CANCEL(int cancel_state;) DCL_LOCK_STATE; #ifdef DEBUG_THREADS GC_log_printf("Called GC_thread_exit_proc on %p,gc_thread=%p\n", (void*)((GC_thread)arg)->id,arg); #endif LOCK(); DISABLE_CANCEL(cancel_state); GC_wait_for_gc_completion(FALSE); GC_unregister_my_thread_inner((GC_thread)arg); RESTORE_CANCEL(cancel_state); UNLOCK(); } #if!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PSP2) GC_API int WRAP_FUNC(pthread_join)(pthread_t thread,void**retval) { int result; GC_thread t; DCL_LOCK_STATE; ASSERT_SYMS_INITIALIZED(); LOCK(); t=GC_lookup_thread(thread); UNLOCK(); result=REAL_FUNC(pthread_join)(thread,retval); #if defined(GC_FREEBSD_THREADS) if (result==EINTR)result=0; #endif if (result==0){ LOCK(); if ((t->flags&FINISHED)!=0) GC_delete_gc_thread(t); UNLOCK(); } return result; } GC_API int WRAP_FUNC(pthread_detach)(pthread_t thread) { int result; GC_thread t; DCL_LOCK_STATE; ASSERT_SYMS_INITIALIZED(); LOCK(); t=GC_lookup_thread(thread); UNLOCK(); result=REAL_FUNC(pthread_detach)(thread); if (result==0){ LOCK(); t->flags|=DETACHED; if ((t->flags&FINISHED)!=0){ GC_delete_gc_thread(t); } UNLOCK(); } return result; } #endif #ifndef GC_NO_PTHREAD_CANCEL GC_API int WRAP_FUNC(pthread_cancel)(pthread_t thread) { #ifdef CANCEL_SAFE GC_thread t; DCL_LOCK_STATE; #endif INIT_REAL_SYMS(); #ifdef CANCEL_SAFE LOCK(); t=GC_lookup_thread(thread); if (t!=NULL&&(t->flags&DISABLED_GC)==0){ t->flags|=DISABLED_GC; GC_dont_gc++; } UNLOCK(); #endif return REAL_FUNC(pthread_cancel)(thread); } #endif #ifdef GC_HAVE_PTHREAD_EXIT GC_API GC_PTHREAD_EXIT_ATTRIBUTE void WRAP_FUNC(pthread_exit)(void*retval) { pthread_t self=pthread_self(); GC_thread me; DCL_LOCK_STATE; INIT_REAL_SYMS(); LOCK(); me=GC_lookup_thread(self); if (me!=0&&(me->flags&DISABLED_GC)==0){ me->flags|=DISABLED_GC; GC_dont_gc++; } UNLOCK(); REAL_FUNC(pthread_exit)(retval); } #endif GC_INNER GC_bool GC_in_thread_creation=FALSE; GC_INLINE void GC_record_stack_base(GC_thread me, const struct GC_stack_base*sb) { #ifndef GC_DARWIN_THREADS me->stop_info.stack_ptr=(ptr_t)sb->mem_base; #endif me->stack_end=(ptr_t)sb->mem_base; if (me->stack_end==NULL) ABORT("Bad stack base in GC_register_my_thread"); #ifdef IA64 me->backing_store_end=(ptr_t)sb->reg_base; #endif } STATIC GC_thread GC_register_my_thread_inner(const struct GC_stack_base*sb, pthread_t my_pthread) { GC_thread me; GC_in_thread_creation=TRUE; me=GC_new_thread(my_pthread); GC_in_thread_creation=FALSE; if (me==0) ABORT("Failed to allocate memory for thread registering"); #ifdef GC_DARWIN_THREADS me->stop_info.mach_thread=mach_thread_self(); #endif GC_record_stack_base(me,sb); #ifdef GC_EXPLICIT_SIGNALS_UNBLOCK GC_unblock_gc_signals(); #endif return me; } GC_API void GC_CALL GC_allow_register_threads(void) { GC_ASSERT(GC_lookup_thread(pthread_self())!=0); set_need_to_lock(); } GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base*sb) { pthread_t self=pthread_self(); GC_thread me; DCL_LOCK_STATE; if (GC_need_to_lock==FALSE) ABORT("Threads explicit registering is not previously enabled"); LOCK(); me=GC_lookup_thread(self); if (0==me){ me=GC_register_my_thread_inner(sb,self); #if defined(CPPCHECK) GC_noop1(me->flags); #endif me->flags|=DETACHED; #if defined(THREAD_LOCAL_ALLOC) GC_init_thread_local(&(me->tlfs)); #endif UNLOCK(); return GC_SUCCESS; } else if ((me->flags&FINISHED)!=0){ #ifdef GC_DARWIN_THREADS me->stop_info.mach_thread=mach_thread_self(); #endif GC_record_stack_base(me,sb); me->flags&=~FINISHED; #ifdef GC_EXPLICIT_SIGNALS_UNBLOCK GC_unblock_gc_signals(); #endif #if defined(THREAD_LOCAL_ALLOC) GC_init_thread_local(&(me->tlfs)); #endif UNLOCK(); return GC_SUCCESS; } else { UNLOCK(); return GC_DUPLICATE; } } struct start_info { void*(*start_routine)(void*); void*arg; word flags; sem_t registered; }; GC_INNER_PTHRSTART GC_thread GC_start_rtn_prepare_thread( void*(**pstart)(void*), void**pstart_arg, struct GC_stack_base*sb,void*arg) { struct start_info*si=(struct start_info*)arg; pthread_t self=pthread_self(); GC_thread me; DCL_LOCK_STATE; #ifdef DEBUG_THREADS GC_log_printf("Starting thread %p,pid=%ld,sp=%p\n", (void*)self,(long)getpid(),(void*)&arg); #endif LOCK(); me=GC_register_my_thread_inner(sb,self); me->flags=si->flags; #if defined(THREAD_LOCAL_ALLOC) GC_init_thread_local(&(me->tlfs)); #endif UNLOCK(); *pstart=si->start_routine; #ifdef DEBUG_THREADS GC_log_printf("start_routine=%p\n",(void*)(signed_word)(*pstart)); #endif *pstart_arg=si->arg; sem_post(&(si->registered)); return me; } #if!defined(SN_TARGET_ORBIS)&&!defined(SN_TARGET_PSP2) STATIC void*GC_start_routine(void*arg) { #ifdef INCLUDE_LINUX_THREAD_DESCR struct GC_stack_base sb; #ifdef REDIRECT_MALLOC GC_disable(); #endif if (GC_get_stack_base(&sb)!=GC_SUCCESS) ABORT("Failed to get thread stack base"); #ifdef REDIRECT_MALLOC GC_enable(); #endif return GC_inner_start_routine(&sb,arg); #else return GC_call_with_stack_base(GC_inner_start_routine,arg); #endif } GC_API int WRAP_FUNC(pthread_create)(pthread_t*new_thread, GC_PTHREAD_CREATE_CONST pthread_attr_t*attr, void*(*start_routine)(void*),void*arg) { int result; int detachstate; word my_flags=0; struct start_info si; DCL_LOCK_STATE; INIT_REAL_SYMS(); if (!EXPECT(parallel_initialized,TRUE)) GC_init_parallel(); if (sem_init(&si.registered,GC_SEM_INIT_PSHARED,0)!=0) ABORT("sem_init failed"); si.start_routine=start_routine; si.arg=arg; LOCK(); if (!EXPECT(GC_thr_initialized,TRUE)) GC_thr_init(); #ifdef GC_ASSERTIONS { size_t stack_size=0; if (NULL!=attr){ if (pthread_attr_getstacksize(attr,&stack_size)!=0) ABORT("pthread_attr_getstacksize failed"); } if (0==stack_size){ pthread_attr_t my_attr; if (pthread_attr_init(&my_attr)!=0) ABORT("pthread_attr_init failed"); if (pthread_attr_getstacksize(&my_attr,&stack_size)!=0) ABORT("pthread_attr_getstacksize failed"); (void)pthread_attr_destroy(&my_attr); } if (0==stack_size){ #ifndef SOLARIS WARN("Failed to get stack size for assertion checking\n",0); #endif stack_size=1000000; } GC_ASSERT(stack_size>=65536); } #endif if (NULL==attr){ detachstate=PTHREAD_CREATE_JOINABLE; } else { pthread_attr_getdetachstate(attr,&detachstate); } if (PTHREAD_CREATE_DETACHED==detachstate)my_flags|=DETACHED; si.flags=my_flags; UNLOCK(); #ifdef DEBUG_THREADS GC_log_printf("About to start new thread from thread %p\n", (void*)pthread_self()); #endif set_need_to_lock(); result=REAL_FUNC(pthread_create)(new_thread,attr,GC_start_routine, &si); if (0==result){ IF_CANCEL(int cancel_state;) #ifdef DEBUG_THREADS GC_log_printf("Started thread %p\n",(void*)(*new_thread)); #endif DISABLE_CANCEL(cancel_state); while (0!=sem_wait(&si.registered)){ #if defined(GC_HAIKU_THREADS) if (EACCES==errno)continue; #endif if (EINTR!=errno)ABORT("sem_wait failed"); } RESTORE_CANCEL(cancel_state); } sem_destroy(&si.registered); return(result); } #endif #if defined(USE_SPIN_LOCK)||!defined(NO_PTHREAD_TRYLOCK) #define GC_PAUSE_SPIN_CYCLES 10 STATIC void GC_pause(void) { int i; for (i=0;i < GC_PAUSE_SPIN_CYCLES;++i){ #if defined(AO_HAVE_compiler_barrier)&&!defined(BASE_ATOMIC_OPS_EMULATED) AO_compiler_barrier(); #else GC_noop1(i); #endif } } #endif #ifndef SPIN_MAX #define SPIN_MAX 128 #endif GC_INNER volatile GC_bool GC_collecting=FALSE; #if (!defined(USE_SPIN_LOCK)&&!defined(NO_PTHREAD_TRYLOCK))||defined(PARALLEL_MARK) #ifdef LOCK_STATS volatile AO_t GC_spin_count=0; volatile AO_t GC_block_count=0; volatile AO_t GC_unlocked_count=0; #endif STATIC void GC_generic_lock(pthread_mutex_t*lock) { #ifndef NO_PTHREAD_TRYLOCK unsigned pause_length=1; unsigned i; if (0==pthread_mutex_trylock(lock)){ #ifdef LOCK_STATS (void)AO_fetch_and_add1(&GC_unlocked_count); #endif return; } for (;pause_length<=SPIN_MAX;pause_length<<=1){ for (i=0;i < pause_length;++i){ GC_pause(); } switch(pthread_mutex_trylock(lock)){ case 0: #ifdef LOCK_STATS (void)AO_fetch_and_add1(&GC_spin_count); #endif return; case EBUSY: break; default: ABORT("Unexpected error from pthread_mutex_trylock"); } } #endif #ifdef LOCK_STATS (void)AO_fetch_and_add1(&GC_block_count); #endif pthread_mutex_lock(lock); } #endif #if defined(AO_HAVE_char_load)&&!defined(BASE_ATOMIC_OPS_EMULATED) #define is_collecting()((GC_bool)AO_char_load((unsigned char*)&GC_collecting)) #else #define is_collecting()GC_collecting #endif #if defined(USE_SPIN_LOCK) GC_INNER volatile AO_TS_t GC_allocate_lock=AO_TS_INITIALIZER; #define low_spin_max 30 #define high_spin_max SPIN_MAX static volatile AO_t spin_max=low_spin_max; static volatile AO_t last_spins=0; GC_INNER void GC_lock(void) { unsigned my_spin_max; unsigned my_last_spins; unsigned i; if (AO_test_and_set_acquire(&GC_allocate_lock)==AO_TS_CLEAR){ return; } my_spin_max=(unsigned)AO_load(&spin_max); my_last_spins=(unsigned)AO_load(&last_spins); for (i=0;i < my_spin_max;i++){ if (is_collecting()||GC_nprocs==1) goto yield; if (i < my_last_spins/2){ GC_pause(); continue; } if (AO_test_and_set_acquire(&GC_allocate_lock)==AO_TS_CLEAR){ AO_store(&last_spins,(AO_t)i); AO_store(&spin_max,(AO_t)high_spin_max); return; } } AO_store(&spin_max,(AO_t)low_spin_max); yield: for (i=0;;++i){ if (AO_test_and_set_acquire(&GC_allocate_lock)==AO_TS_CLEAR){ return; } #define SLEEP_THRESHOLD 12 if (i < SLEEP_THRESHOLD){ sched_yield(); } else { struct timespec ts; if (i > 24)i=24; ts.tv_sec=0; ts.tv_nsec=1< 2||(glibc_major==2&&glibc_minor>=19)){ if (0!=pthread_mutexattr_init(&mattr)){ ABORT("pthread_mutexattr_init failed"); } if (0!=pthread_mutexattr_settype(&mattr,PTHREAD_MUTEX_NORMAL)){ ABORT("pthread_mutexattr_settype failed"); } if (0!=pthread_mutex_init(&mark_mutex,&mattr)){ ABORT("pthread_mutex_init failed"); } (void)pthread_mutexattr_destroy(&mattr); } #endif } GC_INNER void GC_acquire_mark_lock(void) { #if defined(NUMERIC_THREAD_ID_UNIQUE)&&!defined(THREAD_SANITIZER) GC_ASSERT(GC_mark_lock_holder!=NUMERIC_THREAD_ID(pthread_self())); #endif GC_generic_lock(&mark_mutex); SET_MARK_LOCK_HOLDER; } GC_INNER void GC_release_mark_lock(void) { UNSET_MARK_LOCK_HOLDER; if (pthread_mutex_unlock(&mark_mutex)!=0){ ABORT("pthread_mutex_unlock failed"); } } STATIC void GC_wait_builder(void) { ASSERT_CANCEL_DISABLED(); UNSET_MARK_LOCK_HOLDER; if (pthread_cond_wait(&builder_cv,&mark_mutex)!=0){ ABORT("pthread_cond_wait failed"); } GC_ASSERT(GC_mark_lock_holder==NO_THREAD); SET_MARK_LOCK_HOLDER; } GC_INNER void GC_wait_for_reclaim(void) { GC_acquire_mark_lock(); while (GC_fl_builder_count > 0){ GC_wait_builder(); } GC_release_mark_lock(); } GC_INNER void GC_notify_all_builder(void) { GC_ASSERT(GC_mark_lock_holder==NUMERIC_THREAD_ID(pthread_self())); if (pthread_cond_broadcast(&builder_cv)!=0){ ABORT("pthread_cond_broadcast failed"); } } GC_INNER void GC_wait_marker(void) { ASSERT_CANCEL_DISABLED(); GC_ASSERT(GC_parallel); UNSET_MARK_LOCK_HOLDER; if (pthread_cond_wait(&mark_cv,&mark_mutex)!=0){ ABORT("pthread_cond_wait failed"); } GC_ASSERT(GC_mark_lock_holder==NO_THREAD); SET_MARK_LOCK_HOLDER; } GC_INNER void GC_notify_all_marker(void) { GC_ASSERT(GC_parallel); if (pthread_cond_broadcast(&mark_cv)!=0){ ABORT("pthread_cond_broadcast failed"); } } #endif #ifdef PTHREAD_REGISTER_CANCEL_WEAK_STUBS EXTERN_C_BEGIN extern void __pthread_register_cancel(void)__attribute__((__weak__)); extern void __pthread_unregister_cancel(void)__attribute__((__weak__)); EXTERN_C_END void __pthread_register_cancel(void){} void __pthread_unregister_cancel(void){} #endif #endif #if defined(USE_CUSTOM_SPECIFIC) static const tse invalid_tse={INVALID_QTID,0,0,INVALID_THREADID}; GC_INNER int GC_key_create_inner(tsd**key_ptr) { int i; int ret; tsd*result; GC_ASSERT(I_HOLD_LOCK()); GC_ASSERT((word)(&invalid_tse.next)% sizeof(tse*)==0); result=(tsd*)MALLOC_CLEAR(sizeof(tsd)); if (NULL==result)return ENOMEM; ret=pthread_mutex_init(&result->lock,NULL); if (ret!=0)return ret; for (i=0;i < TS_CACHE_SIZE;++i){ result->cache[i]=( tse*)&invalid_tse; } #ifdef GC_ASSERTIONS for (i=0;i < TS_HASH_SIZE;++i){ GC_ASSERT(result->hash[i].p==0); } #endif *key_ptr=result; return 0; } GC_INNER int GC_setspecific(tsd*key,void*value) { pthread_t self=pthread_self(); unsigned hash_val=HASH(self); volatile tse*entry; GC_ASSERT(I_HOLD_LOCK()); GC_ASSERT(self!=INVALID_THREADID); GC_dont_gc++; entry=(volatile tse*)MALLOC_CLEAR(sizeof(tse)); GC_dont_gc--; if (0==entry)return ENOMEM; pthread_mutex_lock(&(key->lock)); entry->next=key->hash[hash_val].p; entry->thread=self; entry->value=TS_HIDE_VALUE(value); GC_ASSERT(entry->qtid==INVALID_QTID); AO_store_release(&key->hash[hash_val].ao,(AO_t)entry); GC_dirty(( void*)entry); GC_dirty(key->hash+hash_val); if (pthread_mutex_unlock(&key->lock)!=0) ABORT("pthread_mutex_unlock failed (setspecific)"); return 0; } GC_INNER void GC_remove_specific_after_fork(tsd*key,pthread_t t) { unsigned hash_val=HASH(t); tse*entry; tse*prev=NULL; #ifdef CAN_HANDLE_FORK GC_ASSERT(I_HOLD_LOCK()); #endif pthread_mutex_lock(&(key->lock)); entry=key->hash[hash_val].p; while (entry!=NULL&&!THREAD_EQUAL(entry->thread,t)){ prev=entry; entry=entry->next; } if (entry!=NULL){ entry->qtid=INVALID_QTID; if (NULL==prev){ key->hash[hash_val].p=entry->next; GC_dirty(key->hash+hash_val); } else { prev->next=entry->next; GC_dirty(prev); } } #ifdef LINT2 GC_noop1((word)entry); #endif if (pthread_mutex_unlock(&key->lock)!=0) ABORT("pthread_mutex_unlock failed (remove_specific after fork)"); } GC_INNER void*GC_slow_getspecific(tsd*key,word qtid, tse*volatile*cache_ptr) { pthread_t self=pthread_self(); tse*entry=key->hash[HASH(self)].p; GC_ASSERT(qtid!=INVALID_QTID); while (entry!=NULL&&!THREAD_EQUAL(entry->thread,self)){ entry=entry->next; } if (entry==NULL)return NULL; entry->qtid=(AO_t)qtid; *cache_ptr=entry; return TS_REVEAL_PTR(entry->value); } #ifdef GC_ASSERTIONS void GC_check_tsd_marks(tsd*key) { int i; tse*p; if (!GC_is_marked(GC_base(key))){ ABORT("Unmarked thread-specific-data table"); } for (i=0;i < TS_HASH_SIZE;++i){ for (p=key->hash[i].p;p!=0;p=p->next){ if (!GC_is_marked(GC_base(p))){ ABORT_ARG1("Unmarked thread-specific-data entry", " at %p",(void*)p); } } } for (i=0;i < TS_CACHE_SIZE;++i){ p=key->cache[i]; if (p!=&invalid_tse&&!GC_is_marked(GC_base(p))){ ABORT_ARG1("Unmarked cached thread-specific-data entry", " at %p",(void*)p); } } } #endif #endif #if defined(GC_WIN32_THREADS) #ifdef THREAD_LOCAL_ALLOC #endif #if!defined(USE_PTHREAD_LOCKS) GC_INNER CRITICAL_SECTION GC_allocate_ml; #ifdef GC_ASSERTIONS GC_INNER DWORD GC_lock_holder=NO_THREAD; #endif #else GC_INNER pthread_mutex_t GC_allocate_ml=PTHREAD_MUTEX_INITIALIZER; #ifdef GC_ASSERTIONS GC_INNER unsigned long GC_lock_holder=NO_THREAD; #endif #endif #undef CreateThread #undef ExitThread #undef _beginthreadex #undef _endthreadex #ifdef GC_PTHREADS #include #undef pthread_create #undef pthread_join #undef pthread_detach #ifndef GC_NO_PTHREAD_SIGMASK #undef pthread_sigmask #endif STATIC void*GC_pthread_start(void*arg); STATIC void GC_thread_exit_proc(void*arg); #include #ifdef CAN_CALL_ATFORK #include #endif #elif!defined(MSWINCE) #include #include #endif static ptr_t copy_ptr_regs(word*regs,const CONTEXT*pcontext); #if defined(I386) #ifdef WOW64_THREAD_CONTEXT_WORKAROUND #define PUSHED_REGS_COUNT 9 #else #define PUSHED_REGS_COUNT 7 #endif #elif defined(X86_64)||defined(SHx) #define PUSHED_REGS_COUNT 15 #elif defined(ARM32) #define PUSHED_REGS_COUNT 13 #elif defined(AARCH64) #define PUSHED_REGS_COUNT 30 #elif defined(MIPS)||defined(ALPHA) #define PUSHED_REGS_COUNT 28 #elif defined(PPC) #define PUSHED_REGS_COUNT 29 #endif #if (defined(GC_DLL)||defined(GC_INSIDE_DLL))&&!defined(NO_CRT)&&!defined(GC_NO_THREADS_DISCOVERY)&&!defined(MSWINCE)&&!defined(THREAD_LOCAL_ALLOC)&&!defined(GC_PTHREADS) #ifdef GC_DISCOVER_TASK_THREADS #define GC_win32_dll_threads TRUE #else STATIC GC_bool GC_win32_dll_threads=FALSE; #endif #else #ifndef GC_NO_THREADS_DISCOVERY #define GC_NO_THREADS_DISCOVERY #endif #define GC_win32_dll_threads FALSE #undef MAX_THREADS #define MAX_THREADS 1 #endif typedef LONG*IE_t; STATIC GC_bool GC_thr_initialized=FALSE; #ifndef GC_ALWAYS_MULTITHREADED GC_INNER GC_bool GC_need_to_lock=FALSE; #endif static GC_bool parallel_initialized=FALSE; GC_API void GC_CALL GC_use_threads_discovery(void) { #ifdef GC_NO_THREADS_DISCOVERY ABORT("GC DllMain-based thread registration unsupported"); #else GC_ASSERT(!parallel_initialized); #ifndef GC_DISCOVER_TASK_THREADS GC_win32_dll_threads=TRUE; #endif GC_init_parallel(); #endif } #define ADDR_LIMIT ((ptr_t)GC_WORD_MAX) struct GC_Thread_Rep { union { #ifndef GC_NO_THREADS_DISCOVERY volatile AO_t in_use; LONG long_in_use; #endif struct GC_Thread_Rep*next; } tm; DWORD id; #ifdef MSWINCE #define THREAD_HANDLE(t)(HANDLE)(word)(t)->id #else HANDLE handle; #define THREAD_HANDLE(t)(t)->handle #endif ptr_t stack_base; ptr_t last_stack_min; #ifdef IA64 ptr_t backing_store_end; ptr_t backing_store_ptr; #elif defined(I386) ptr_t initial_stack_base; #endif ptr_t thread_blocked_sp; struct GC_traced_stack_sect_s*traced_stack_sect; unsigned short finalizer_skipped; unsigned char finalizer_nested; unsigned char suspended; #ifdef GC_PTHREADS unsigned char flags; #define FINISHED 1 #define DETACHED 2 #define KNOWN_FINISHED(t)(((t)->flags)&FINISHED) pthread_t pthread_id; void*status; #else #define KNOWN_FINISHED(t)0 #endif #ifdef THREAD_LOCAL_ALLOC struct thread_local_freelists tlfs; #endif #ifdef RETRY_GET_THREAD_CONTEXT ptr_t context_sp; word context_regs[PUSHED_REGS_COUNT]; #endif }; typedef struct GC_Thread_Rep*GC_thread; typedef volatile struct GC_Thread_Rep*GC_vthread; #ifndef GC_NO_THREADS_DISCOVERY STATIC DWORD GC_main_thread=0; STATIC volatile AO_t GC_attached_thread=FALSE; STATIC volatile GC_bool GC_please_stop=FALSE; #elif defined(GC_ASSERTIONS) STATIC GC_bool GC_please_stop=FALSE; #endif #if defined(WRAP_MARK_SOME)&&!defined(GC_PTHREADS) GC_INNER GC_bool GC_started_thread_while_stopped(void) { #ifndef GC_NO_THREADS_DISCOVERY if (GC_win32_dll_threads){ #ifdef AO_HAVE_compare_and_swap_release if (AO_compare_and_swap_release(&GC_attached_thread,TRUE, FALSE)) return TRUE; #else AO_nop_full(); if (AO_load(&GC_attached_thread)){ AO_store(&GC_attached_thread,FALSE); return TRUE; } #endif } #endif return FALSE; } #endif #ifndef MAX_THREADS #define MAX_THREADS 512 #endif volatile struct GC_Thread_Rep dll_thread_table[MAX_THREADS]; STATIC volatile LONG GC_max_thread_index=0; #ifndef THREAD_TABLE_SZ #define THREAD_TABLE_SZ 256 #endif #define THREAD_TABLE_INDEX(id)(int)((((id)>>8)^(id))% THREAD_TABLE_SZ) STATIC GC_thread GC_threads[THREAD_TABLE_SZ]; static struct GC_Thread_Rep first_thread; static GC_bool first_thread_used=FALSE; STATIC GC_thread GC_new_thread(DWORD id) { int hv=THREAD_TABLE_INDEX(id); GC_thread result; #ifdef DEBUG_THREADS GC_log_printf("Creating thread 0x%lx\n",(long)id); if (GC_threads[hv]!=NULL) GC_log_printf("Hash collision at GC_threads[%d]\n",hv); #endif GC_ASSERT(I_HOLD_LOCK()); if (!EXPECT(first_thread_used,TRUE)){ result=&first_thread; first_thread_used=TRUE; GC_ASSERT(NULL==GC_threads[hv]); } else { GC_ASSERT(!GC_win32_dll_threads); result=(struct GC_Thread_Rep*) GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep),NORMAL); if (result==0)return(0); } result->tm.next=GC_threads[hv]; GC_threads[hv]=result; #ifdef GC_PTHREADS GC_ASSERT(result->flags==0); #endif GC_ASSERT(result->thread_blocked_sp==NULL); if (EXPECT(result!=&first_thread,TRUE)) GC_dirty(result); return(result); } GC_INNER GC_bool GC_in_thread_creation=FALSE; GC_INLINE void GC_record_stack_base(GC_vthread me, const struct GC_stack_base*sb) { me->stack_base=(ptr_t)sb->mem_base; #ifdef IA64 me->backing_store_end=(ptr_t)sb->reg_base; #elif defined(I386) me->initial_stack_base=(ptr_t)sb->mem_base; #endif if (me->stack_base==NULL) ABORT("Bad stack base in GC_register_my_thread"); } STATIC GC_thread GC_register_my_thread_inner(const struct GC_stack_base*sb, DWORD thread_id) { GC_vthread me; #if defined(MPROTECT_VDB)&&!defined(CYGWIN32) if (GC_auto_incremental #ifdef GWW_VDB &&!GC_gww_dirty_init() #endif ) GC_set_write_fault_handler(); #endif #ifndef GC_NO_THREADS_DISCOVERY if (GC_win32_dll_threads){ int i; for (i=0; InterlockedExchange(&dll_thread_table[i].tm.long_in_use,1)!=0; i++){ if (i==MAX_THREADS - 1) ABORT("Too many threads"); } while (i > GC_max_thread_index){ InterlockedIncrement((IE_t)&GC_max_thread_index); } if (GC_max_thread_index>=MAX_THREADS){ GC_max_thread_index=MAX_THREADS - 1; } me=dll_thread_table+i; } else #endif { GC_ASSERT(I_HOLD_LOCK()); GC_in_thread_creation=TRUE; me=GC_new_thread(thread_id); GC_in_thread_creation=FALSE; if (me==0) ABORT("Failed to allocate memory for thread registering"); } #ifdef GC_PTHREADS me->pthread_id=pthread_self(); #endif #ifndef MSWINCE if (!DuplicateHandle(GetCurrentProcess(),GetCurrentThread(), GetCurrentProcess(), (HANDLE*)&(me->handle), 0,FALSE, DUPLICATE_SAME_ACCESS)){ ABORT_ARG1("DuplicateHandle failed", ":errcode=0x%X",(unsigned)GetLastError()); } #endif me->last_stack_min=ADDR_LIMIT; GC_record_stack_base(me,sb); me->id=thread_id; #if defined(THREAD_LOCAL_ALLOC) GC_init_thread_local((GC_tlfs)(&(me->tlfs))); #endif #ifndef GC_NO_THREADS_DISCOVERY if (GC_win32_dll_threads){ if (GC_please_stop){ AO_store(&GC_attached_thread,TRUE); AO_nop_full(); } } else #endif { GC_ASSERT(!GC_please_stop); } return (GC_thread)(me); } GC_INLINE LONG GC_get_max_thread_index(void) { LONG my_max=GC_max_thread_index; if (my_max>=MAX_THREADS)return MAX_THREADS - 1; return my_max; } STATIC GC_thread GC_lookup_thread_inner(DWORD thread_id) { #ifndef GC_NO_THREADS_DISCOVERY if (GC_win32_dll_threads){ int i; LONG my_max=GC_get_max_thread_index(); for (i=0;i<=my_max&& (!AO_load_acquire(&dll_thread_table[i].tm.in_use) ||dll_thread_table[i].id!=thread_id); i++){ } return i<=my_max?(GC_thread)(dll_thread_table+i):NULL; } else #endif { GC_thread p=GC_threads[THREAD_TABLE_INDEX(thread_id)]; GC_ASSERT(I_HOLD_LOCK()); while (p!=0&&p->id!=thread_id)p=p->tm.next; return(p); } } #ifdef LINT2 #define CHECK_LOOKUP_MY_THREAD(me)if (!(me))ABORT("GC_lookup_thread_inner(GetCurrentThreadId)failed") #else #define CHECK_LOOKUP_MY_THREAD(me) #endif GC_INNER void GC_reset_finalizer_nested(void) { GC_thread me=GC_lookup_thread_inner(GetCurrentThreadId()); CHECK_LOOKUP_MY_THREAD(me); me->finalizer_nested=0; } GC_INNER unsigned char*GC_check_finalizer_nested(void) { GC_thread me=GC_lookup_thread_inner(GetCurrentThreadId()); unsigned nesting_level; CHECK_LOOKUP_MY_THREAD(me); nesting_level=me->finalizer_nested; if (nesting_level){ if (++me->finalizer_skipped < (1U<finalizer_skipped=0; } me->finalizer_nested=(unsigned char)(nesting_level+1); return&me->finalizer_nested; } #if defined(GC_ASSERTIONS)&&defined(THREAD_LOCAL_ALLOC) GC_bool GC_is_thread_tsd_valid(void*tsd) { GC_thread me; DCL_LOCK_STATE; LOCK(); me=GC_lookup_thread_inner(GetCurrentThreadId()); UNLOCK(); return (word)tsd>=(word)(&me->tlfs) &&(word)tsd < (word)(&me->tlfs)+sizeof(me->tlfs); } #endif GC_API int GC_CALL GC_thread_is_registered(void) { DWORD thread_id=GetCurrentThreadId(); GC_thread me; DCL_LOCK_STATE; LOCK(); me=GC_lookup_thread_inner(thread_id); UNLOCK(); return me!=NULL; } GC_API void GC_CALL GC_register_altstack(void*stack GC_ATTR_UNUSED, GC_word stack_size GC_ATTR_UNUSED, void*altstack GC_ATTR_UNUSED, GC_word altstack_size GC_ATTR_UNUSED) { } #if defined(MPROTECT_VDB) #define UNPROTECT_THREAD(t)if (!GC_win32_dll_threads&&GC_auto_incremental&&t!=&first_thread){ GC_ASSERT(SMALL_OBJ(GC_size(t)));GC_remove_protection(HBLKPTR(t),1,FALSE);} else (void)0 #else #define UNPROTECT_THREAD(t)(void)0 #endif #ifdef CYGWIN32 #define GC_PTHREAD_PTRVAL(pthread_id)pthread_id #elif defined(GC_WIN32_PTHREADS)||defined(GC_PTHREADS_PARAMARK) #include #if defined(__WINPTHREADS_VERSION_MAJOR) #define GC_PTHREAD_PTRVAL(pthread_id)pthread_id #else #define GC_PTHREAD_PTRVAL(pthread_id)pthread_id.p #endif #endif STATIC void GC_delete_gc_thread_no_free(GC_vthread t) { #ifndef MSWINCE CloseHandle(t->handle); #endif #ifndef GC_NO_THREADS_DISCOVERY if (GC_win32_dll_threads){ t->stack_base=0; t->id=0; t->suspended=FALSE; #ifdef RETRY_GET_THREAD_CONTEXT t->context_sp=NULL; #endif AO_store_release(&t->tm.in_use,FALSE); } else #endif { DWORD id=((GC_thread)t)->id; int hv=THREAD_TABLE_INDEX(id); GC_thread p=GC_threads[hv]; GC_thread prev=NULL; GC_ASSERT(I_HOLD_LOCK()); while (p!=(GC_thread)t){ prev=p; p=p->tm.next; } if (prev==0){ GC_threads[hv]=p->tm.next; } else { GC_ASSERT(prev!=&first_thread); prev->tm.next=p->tm.next; GC_dirty(prev); } } } STATIC void GC_delete_thread(DWORD id) { if (GC_win32_dll_threads){ GC_vthread t=GC_lookup_thread_inner(id); if (0==t){ WARN("Removing nonexistent thread,id=%" WARN_PRIdPTR "\n",id); } else { GC_delete_gc_thread_no_free(t); } } else { int hv=THREAD_TABLE_INDEX(id); GC_thread p=GC_threads[hv]; GC_thread prev=NULL; GC_ASSERT(I_HOLD_LOCK()); while (p->id!=id){ prev=p; p=p->tm.next; } #ifndef MSWINCE CloseHandle(p->handle); #endif if (prev==0){ GC_threads[hv]=p->tm.next; } else { GC_ASSERT(prev!=&first_thread); prev->tm.next=p->tm.next; GC_dirty(prev); } if (EXPECT(p!=&first_thread,TRUE)){ GC_INTERNAL_FREE(p); } } } GC_API void GC_CALL GC_allow_register_threads(void) { GC_ASSERT(GC_lookup_thread_inner(GetCurrentThreadId())!=0); #if!defined(GC_ALWAYS_MULTITHREADED)&&!defined(PARALLEL_MARK)&&!defined(GC_NO_THREADS_DISCOVERY) parallel_initialized=TRUE; #endif set_need_to_lock(); } GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base*sb) { GC_thread me; DWORD thread_id=GetCurrentThreadId(); DCL_LOCK_STATE; if (GC_need_to_lock==FALSE) ABORT("Threads explicit registering is not previously enabled"); LOCK(); me=GC_lookup_thread_inner(thread_id); if (me==0){ #ifdef GC_PTHREADS me=GC_register_my_thread_inner(sb,thread_id); #if defined(CPPCHECK) GC_noop1(me->flags); #endif me->flags|=DETACHED; #else GC_register_my_thread_inner(sb,thread_id); #endif UNLOCK(); return GC_SUCCESS; } else #ifdef GC_PTHREADS if ((me->flags&FINISHED)!=0){ GC_record_stack_base(me,sb); me->flags&=~FINISHED; #ifdef THREAD_LOCAL_ALLOC GC_init_thread_local((GC_tlfs)(&me->tlfs)); #endif UNLOCK(); return GC_SUCCESS; } else #endif { UNLOCK(); return GC_DUPLICATE; } } STATIC void GC_wait_for_gc_completion(GC_bool wait_for_all) { GC_ASSERT(I_HOLD_LOCK()); if (GC_incremental&&GC_collection_in_progress()){ word old_gc_no=GC_gc_no; do { ENTER_GC(); GC_in_thread_creation=TRUE; GC_collect_a_little_inner(1); GC_in_thread_creation=FALSE; EXIT_GC(); UNLOCK(); Sleep(0); LOCK(); } while (GC_incremental&&GC_collection_in_progress() &&(wait_for_all||old_gc_no==GC_gc_no)); } } GC_API int GC_CALL GC_unregister_my_thread(void) { DCL_LOCK_STATE; #ifdef DEBUG_THREADS GC_log_printf("Unregistering thread 0x%lx\n",(long)GetCurrentThreadId()); #endif if (GC_win32_dll_threads){ #if defined(THREAD_LOCAL_ALLOC) GC_ASSERT(FALSE); #else GC_delete_thread(GetCurrentThreadId()); #endif } else { #if defined(THREAD_LOCAL_ALLOC)||defined(GC_PTHREADS) GC_thread me; #endif DWORD thread_id=GetCurrentThreadId(); LOCK(); GC_wait_for_gc_completion(FALSE); #if defined(THREAD_LOCAL_ALLOC)||defined(GC_PTHREADS) me=GC_lookup_thread_inner(thread_id); CHECK_LOOKUP_MY_THREAD(me); GC_ASSERT(!KNOWN_FINISHED(me)); #endif #if defined(THREAD_LOCAL_ALLOC) GC_ASSERT(GC_getspecific(GC_thread_key)==&me->tlfs); GC_destroy_thread_local(&(me->tlfs)); #endif #ifdef GC_PTHREADS if ((me->flags&DETACHED)==0){ me->flags|=FINISHED; } else #endif { GC_delete_thread(thread_id); } #if defined(THREAD_LOCAL_ALLOC) GC_remove_specific(GC_thread_key); #endif UNLOCK(); } return GC_SUCCESS; } GC_INNER void GC_do_blocking_inner(ptr_t data,void*context GC_ATTR_UNUSED) { struct blocking_data*d=(struct blocking_data*)data; DWORD thread_id=GetCurrentThreadId(); GC_thread me; #ifdef IA64 ptr_t stack_ptr=GC_save_regs_in_stack(); #endif DCL_LOCK_STATE; LOCK(); me=GC_lookup_thread_inner(thread_id); CHECK_LOOKUP_MY_THREAD(me); GC_ASSERT(me->thread_blocked_sp==NULL); #ifdef IA64 me->backing_store_ptr=stack_ptr; #endif me->thread_blocked_sp=(ptr_t)&d; UNLOCK(); d->client_data=(d->fn)(d->client_data); LOCK(); #if defined(CPPCHECK) GC_noop1((word)me->thread_blocked_sp); #endif me->thread_blocked_sp=NULL; UNLOCK(); } GC_API void*GC_CALL GC_call_with_gc_active(GC_fn_type fn, void*client_data) { struct GC_traced_stack_sect_s stacksect; DWORD thread_id=GetCurrentThreadId(); GC_thread me; DCL_LOCK_STATE; LOCK(); me=GC_lookup_thread_inner(thread_id); CHECK_LOOKUP_MY_THREAD(me); GC_ASSERT(me->stack_base!=NULL); if ((word)me->stack_base < (word)(&stacksect)){ me->stack_base=(ptr_t)(&stacksect); #if defined(I386) me->initial_stack_base=me->stack_base; #endif } if (me->thread_blocked_sp==NULL){ UNLOCK(); client_data=fn(client_data); GC_noop1(COVERT_DATAFLOW(&stacksect)); return client_data; } stacksect.saved_stack_ptr=me->thread_blocked_sp; #ifdef IA64 stacksect.backing_store_end=GC_save_regs_in_stack(); stacksect.saved_backing_store_ptr=me->backing_store_ptr; #endif stacksect.prev=me->traced_stack_sect; me->thread_blocked_sp=NULL; me->traced_stack_sect=&stacksect; UNLOCK(); client_data=fn(client_data); GC_ASSERT(me->thread_blocked_sp==NULL); GC_ASSERT(me->traced_stack_sect==&stacksect); LOCK(); #if defined(CPPCHECK) GC_noop1((word)me->traced_stack_sect); #endif me->traced_stack_sect=stacksect.prev; #ifdef IA64 me->backing_store_ptr=stacksect.saved_backing_store_ptr; #endif me->thread_blocked_sp=stacksect.saved_stack_ptr; UNLOCK(); return client_data; } GC_API void GC_CALL GC_set_stackbottom(void*gc_thread_handle, const struct GC_stack_base*sb) { GC_thread t=(GC_thread)gc_thread_handle; GC_ASSERT(sb->mem_base!=NULL); if (!EXPECT(GC_is_initialized,TRUE)){ GC_ASSERT(NULL==t); GC_stackbottom=(char*)sb->mem_base; #ifdef IA64 GC_register_stackbottom=(ptr_t)sb->reg_base; #endif return; } GC_ASSERT(I_HOLD_LOCK()); if (NULL==t){ t=GC_lookup_thread_inner(GetCurrentThreadId()); CHECK_LOOKUP_MY_THREAD(t); } GC_ASSERT(!KNOWN_FINISHED(t)); GC_ASSERT(NULL==t->thread_blocked_sp &&NULL==t->traced_stack_sect); t->stack_base=(ptr_t)sb->mem_base; t->last_stack_min=ADDR_LIMIT; #ifdef IA64 t->backing_store_end=(ptr_t)sb->reg_base; #endif } GC_API void*GC_CALL GC_get_my_stackbottom(struct GC_stack_base*sb) { DWORD thread_id=GetCurrentThreadId(); GC_thread me; DCL_LOCK_STATE; LOCK(); me=GC_lookup_thread_inner(thread_id); CHECK_LOOKUP_MY_THREAD(me); sb->mem_base=me->stack_base; #ifdef IA64 sb->reg_base=me->backing_store_end; #endif UNLOCK(); return (void*)me; } #ifdef GC_PTHREADS #define PTHREAD_MAP_SIZE 512 DWORD GC_pthread_map_cache[PTHREAD_MAP_SIZE]={0}; #define PTHREAD_MAP_INDEX(pthread_id)((NUMERIC_THREAD_ID(pthread_id)>>5)% PTHREAD_MAP_SIZE) #define SET_PTHREAD_MAP_CACHE(pthread_id,win32_id)(void)(GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)]=(win32_id)) #define GET_PTHREAD_MAP_CACHE(pthread_id)GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)] STATIC GC_thread GC_lookup_pthread(pthread_t id) { #ifndef GC_NO_THREADS_DISCOVERY if (GC_win32_dll_threads){ int i; LONG my_max=GC_get_max_thread_index(); for (i=0;i<=my_max&& (!AO_load_acquire(&dll_thread_table[i].tm.in_use) ||THREAD_EQUAL(dll_thread_table[i].pthread_id,id)); i++){ } return i<=my_max?(GC_thread)(dll_thread_table+i):NULL; } else #endif { DWORD win32_id=GET_PTHREAD_MAP_CACHE(id); int hv_guess=THREAD_TABLE_INDEX(win32_id); int hv; GC_thread p; DCL_LOCK_STATE; LOCK(); for (p=GC_threads[hv_guess];0!=p;p=p->tm.next){ if (THREAD_EQUAL(p->pthread_id,id)) goto foundit; } for (hv=0;hv < THREAD_TABLE_SZ;++hv){ for (p=GC_threads[hv];0!=p;p=p->tm.next){ if (THREAD_EQUAL(p->pthread_id,id)) goto foundit; } } p=0; foundit: UNLOCK(); return p; } } #endif #ifdef CAN_HANDLE_FORK STATIC void GC_remove_all_threads_but_me(void) { int hv; GC_thread me=NULL; DWORD thread_id; pthread_t pthread_id=pthread_self(); GC_ASSERT(!GC_win32_dll_threads); for (hv=0;hv < THREAD_TABLE_SZ;++hv){ GC_thread p,next; for (p=GC_threads[hv];0!=p;p=next){ next=p->tm.next; if (THREAD_EQUAL(p->pthread_id,pthread_id) &&me==NULL){ me=p; p->tm.next=0; } else { #ifdef THREAD_LOCAL_ALLOC if ((p->flags&FINISHED)==0){ GC_remove_specific_after_fork(GC_thread_key,p->pthread_id); } #endif if (&first_thread!=p) GC_INTERNAL_FREE(p); } } GC_threads[hv]=NULL; } GC_ASSERT(me!=NULL); thread_id=GetCurrentThreadId(); GC_threads[THREAD_TABLE_INDEX(thread_id)]=me; me->id=thread_id; #ifndef MSWINCE if (!DuplicateHandle(GetCurrentProcess(),GetCurrentThread(), GetCurrentProcess(),(HANDLE*)&me->handle, 0,FALSE, DUPLICATE_SAME_ACCESS)) ABORT("DuplicateHandle failed"); #endif #if defined(THREAD_LOCAL_ALLOC)&&!defined(USE_CUSTOM_SPECIFIC) if (GC_setspecific(GC_thread_key,&me->tlfs)!=0) ABORT("GC_setspecific failed (in child)"); #endif } static void fork_prepare_proc(void) { LOCK(); #ifdef PARALLEL_MARK if (GC_parallel) GC_wait_for_reclaim(); #endif GC_wait_for_gc_completion(TRUE); #ifdef PARALLEL_MARK if (GC_parallel) GC_acquire_mark_lock(); #endif } static void fork_parent_proc(void) { #ifdef PARALLEL_MARK if (GC_parallel) GC_release_mark_lock(); #endif UNLOCK(); } static void fork_child_proc(void) { #ifdef PARALLEL_MARK if (GC_parallel){ GC_release_mark_lock(); GC_parallel=FALSE; } #endif GC_remove_all_threads_but_me(); UNLOCK(); } GC_API void GC_CALL GC_atfork_prepare(void) { if (!EXPECT(GC_is_initialized,TRUE))GC_init(); if (GC_handle_fork<=0) fork_prepare_proc(); } GC_API void GC_CALL GC_atfork_parent(void) { if (GC_handle_fork<=0) fork_parent_proc(); } GC_API void GC_CALL GC_atfork_child(void) { if (GC_handle_fork<=0) fork_child_proc(); } #endif void GC_push_thread_structures(void) { GC_ASSERT(I_HOLD_LOCK()); #ifndef GC_NO_THREADS_DISCOVERY if (GC_win32_dll_threads){ } else #endif { GC_PUSH_ALL_SYM(GC_threads); } #if defined(THREAD_LOCAL_ALLOC) GC_PUSH_ALL_SYM(GC_thread_key); #endif } #ifdef WOW64_THREAD_CONTEXT_WORKAROUND #ifndef CONTEXT_EXCEPTION_ACTIVE #define CONTEXT_EXCEPTION_ACTIVE 0x08000000 #define CONTEXT_EXCEPTION_REQUEST 0x40000000 #define CONTEXT_EXCEPTION_REPORTING 0x80000000 #endif static BOOL isWow64; #define GET_THREAD_CONTEXT_FLAGS (isWow64?CONTEXT_INTEGER|CONTEXT_CONTROL|CONTEXT_EXCEPTION_REQUEST|CONTEXT_SEGMENTS:CONTEXT_INTEGER|CONTEXT_CONTROL) #else #define GET_THREAD_CONTEXT_FLAGS (CONTEXT_INTEGER|CONTEXT_CONTROL) #endif STATIC void GC_suspend(GC_thread t) { #ifndef MSWINCE DWORD exitCode; #endif #ifdef RETRY_GET_THREAD_CONTEXT int retry_cnt=0; #define MAX_SUSPEND_THREAD_RETRIES (1000*1000) #endif #ifdef DEBUG_THREADS GC_log_printf("Suspending 0x%x\n",(int)t->id); #endif UNPROTECT_THREAD(t); GC_acquire_dirty_lock(); #ifdef MSWINCE while (SuspendThread(THREAD_HANDLE(t))==(DWORD)-1){ GC_release_dirty_lock(); Sleep(10); GC_acquire_dirty_lock(); } #elif defined(RETRY_GET_THREAD_CONTEXT) for (;;){ if (GetExitCodeThread(t->handle,&exitCode) &&exitCode!=STILL_ACTIVE){ GC_release_dirty_lock(); #ifdef GC_PTHREADS t->stack_base=0; #else GC_ASSERT(GC_win32_dll_threads); GC_delete_gc_thread_no_free(t); #endif return; } if (SuspendThread(t->handle)!=(DWORD)-1){ CONTEXT context; context.ContextFlags=GET_THREAD_CONTEXT_FLAGS; if (GetThreadContext(t->handle,&context)){ t->context_sp=copy_ptr_regs(t->context_regs,&context); break; } if (ResumeThread(t->handle)==(DWORD)-1) ABORT("ResumeThread failed in suspend loop"); } if (retry_cnt > 1){ GC_release_dirty_lock(); Sleep(0); GC_acquire_dirty_lock(); } if (++retry_cnt>=MAX_SUSPEND_THREAD_RETRIES) ABORT("SuspendThread loop failed"); } #else if (GetExitCodeThread(t->handle,&exitCode) &&exitCode!=STILL_ACTIVE){ GC_release_dirty_lock(); #ifdef GC_PTHREADS t->stack_base=0; #else GC_ASSERT(GC_win32_dll_threads); GC_delete_gc_thread_no_free(t); #endif return; } if (SuspendThread(t->handle)==(DWORD)-1) ABORT("SuspendThread failed"); #endif t->suspended=(unsigned char)TRUE; GC_release_dirty_lock(); if (GC_on_thread_event) GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,THREAD_HANDLE(t)); } #if defined(GC_ASSERTIONS)&&((defined(MSWIN32)&&!defined(CONSOLE_LOG))||defined(MSWINCE)) GC_INNER GC_bool GC_write_disabled=FALSE; #endif GC_INNER void GC_stop_world(void) { DWORD thread_id=GetCurrentThreadId(); if (!GC_thr_initialized) ABORT("GC_stop_world()called before GC_thr_init()"); GC_ASSERT(I_HOLD_LOCK()); #ifdef PARALLEL_MARK if (GC_parallel){ GC_acquire_mark_lock(); GC_ASSERT(GC_fl_builder_count==0); } #endif #if!defined(GC_NO_THREADS_DISCOVERY)||defined(GC_ASSERTIONS) GC_please_stop=TRUE; #endif #if (defined(MSWIN32)&&!defined(CONSOLE_LOG))||defined(MSWINCE) GC_ASSERT(!GC_write_disabled); EnterCriticalSection(&GC_write_cs); #ifdef GC_ASSERTIONS GC_write_disabled=TRUE; #endif #endif #ifndef GC_NO_THREADS_DISCOVERY if (GC_win32_dll_threads){ int i; int my_max; AO_store(&GC_attached_thread,FALSE); my_max=(int)GC_get_max_thread_index(); for (i=0;i<=my_max;i++){ GC_vthread t=dll_thread_table+i; if (t->stack_base!=0&&t->thread_blocked_sp==NULL &&t->id!=thread_id){ GC_suspend((GC_thread)t); } } } else #endif { GC_thread t; int i; for (i=0;i < THREAD_TABLE_SZ;i++){ for (t=GC_threads[i];t!=0;t=t->tm.next){ if (t->stack_base!=0&&t->thread_blocked_sp==NULL &&!KNOWN_FINISHED(t)&&t->id!=thread_id){ GC_suspend(t); } } } } #if (defined(MSWIN32)&&!defined(CONSOLE_LOG))||defined(MSWINCE) #ifdef GC_ASSERTIONS GC_write_disabled=FALSE; #endif LeaveCriticalSection(&GC_write_cs); #endif #ifdef PARALLEL_MARK if (GC_parallel) GC_release_mark_lock(); #endif } GC_INNER void GC_start_world(void) { #ifdef GC_ASSERTIONS DWORD thread_id=GetCurrentThreadId(); #endif GC_ASSERT(I_HOLD_LOCK()); if (GC_win32_dll_threads){ LONG my_max=GC_get_max_thread_index(); int i; for (i=0;i<=my_max;i++){ GC_thread t=(GC_thread)(dll_thread_table+i); if (t->suspended){ #ifdef DEBUG_THREADS GC_log_printf("Resuming 0x%x\n",(int)t->id); #endif GC_ASSERT(t->stack_base!=0&&t->id!=thread_id); if (ResumeThread(THREAD_HANDLE(t))==(DWORD)-1) ABORT("ResumeThread failed"); t->suspended=FALSE; if (GC_on_thread_event) GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED,THREAD_HANDLE(t)); } } } else { GC_thread t; int i; for (i=0;i < THREAD_TABLE_SZ;i++){ for (t=GC_threads[i];t!=0;t=t->tm.next){ if (t->suspended){ #ifdef DEBUG_THREADS GC_log_printf("Resuming 0x%x\n",(int)t->id); #endif GC_ASSERT(t->stack_base!=0&&t->id!=thread_id); if (ResumeThread(THREAD_HANDLE(t))==(DWORD)-1) ABORT("ResumeThread failed"); UNPROTECT_THREAD(t); t->suspended=FALSE; if (GC_on_thread_event) GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED,THREAD_HANDLE(t)); } else { #ifdef DEBUG_THREADS GC_log_printf("Not resuming thread 0x%x as it is not suspended\n", (int)t->id); #endif } } } } #if!defined(GC_NO_THREADS_DISCOVERY)||defined(GC_ASSERTIONS) GC_please_stop=FALSE; #endif } #ifdef MSWINCE #define GC_wince_evaluate_stack_min(s)(ptr_t)(((word)(s)- 1)&~(word)0xFFFF) #elif defined(GC_ASSERTIONS) #define GC_dont_query_stack_min FALSE #endif static ptr_t last_address=0; static MEMORY_BASIC_INFORMATION last_info; STATIC ptr_t GC_get_stack_min(ptr_t s) { ptr_t bottom; GC_ASSERT(I_HOLD_LOCK()); if (s!=last_address){ VirtualQuery(s,&last_info,sizeof(last_info)); last_address=s; } do { bottom=(ptr_t)last_info.BaseAddress; VirtualQuery(bottom - 1,&last_info,sizeof(last_info)); last_address=bottom - 1; } while ((last_info.Protect&PAGE_READWRITE) &&!(last_info.Protect&PAGE_GUARD)); return(bottom); } static GC_bool may_be_in_stack(ptr_t s) { GC_ASSERT(I_HOLD_LOCK()); if (s!=last_address){ VirtualQuery(s,&last_info,sizeof(last_info)); last_address=s; } return (last_info.Protect&PAGE_READWRITE) &&!(last_info.Protect&PAGE_GUARD); } static ptr_t copy_ptr_regs(word*regs,const CONTEXT*pcontext){ ptr_t sp; int cnt=0; #define context (*pcontext) #define PUSH1(reg)(regs[cnt++]=(word)pcontext->reg) #define PUSH2(r1,r2)(PUSH1(r1),PUSH1(r2)) #define PUSH4(r1,r2,r3,r4)(PUSH2(r1,r2),PUSH2(r3,r4)) #if defined(I386) #ifdef WOW64_THREAD_CONTEXT_WORKAROUND PUSH2(ContextFlags,SegFs); #endif PUSH4(Edi,Esi,Ebx,Edx),PUSH2(Ecx,Eax),PUSH1(Ebp); sp=(ptr_t)context.Esp; #elif defined(X86_64) PUSH4(Rax,Rcx,Rdx,Rbx);PUSH2(Rbp,Rsi);PUSH1(Rdi); PUSH4(R8,R9,R10,R11);PUSH4(R12,R13,R14,R15); sp=(ptr_t)context.Rsp; #elif defined(ARM32) PUSH4(R0,R1,R2,R3),PUSH4(R4,R5,R6,R7),PUSH4(R8,R9,R10,R11); PUSH1(R12); sp=(ptr_t)context.Sp; #elif defined(AARCH64) PUSH4(X0,X1,X2,X3),PUSH4(X4,X5,X6,X7),PUSH4(X8,X9,X10,X11); PUSH4(X12,X13,X14,X15),PUSH4(X16,X17,X18,X19),PUSH4(X20,X21,X22,X23); PUSH4(X24,X25,X26,X27),PUSH1(X28); PUSH1(Lr); sp=(ptr_t)context.Sp; #elif defined(SHx) PUSH4(R0,R1,R2,R3),PUSH4(R4,R5,R6,R7),PUSH4(R8,R9,R10,R11); PUSH2(R12,R13),PUSH1(R14); sp=(ptr_t)context.R15; #elif defined(MIPS) PUSH4(IntAt,IntV0,IntV1,IntA0),PUSH4(IntA1,IntA2,IntA3,IntT0); PUSH4(IntT1,IntT2,IntT3,IntT4),PUSH4(IntT5,IntT6,IntT7,IntS0); PUSH4(IntS1,IntS2,IntS3,IntS4),PUSH4(IntS5,IntS6,IntS7,IntT8); PUSH4(IntT9,IntK0,IntK1,IntS8); sp=(ptr_t)context.IntSp; #elif defined(PPC) PUSH4(Gpr0,Gpr3,Gpr4,Gpr5),PUSH4(Gpr6,Gpr7,Gpr8,Gpr9); PUSH4(Gpr10,Gpr11,Gpr12,Gpr14),PUSH4(Gpr15,Gpr16,Gpr17,Gpr18); PUSH4(Gpr19,Gpr20,Gpr21,Gpr22),PUSH4(Gpr23,Gpr24,Gpr25,Gpr26); PUSH4(Gpr27,Gpr28,Gpr29,Gpr30),PUSH1(Gpr31); sp=(ptr_t)context.Gpr1; #elif defined(ALPHA) PUSH4(IntV0,IntT0,IntT1,IntT2),PUSH4(IntT3,IntT4,IntT5,IntT6); PUSH4(IntT7,IntS0,IntS1,IntS2),PUSH4(IntS3,IntS4,IntS5,IntFp); PUSH4(IntA0,IntA1,IntA2,IntA3),PUSH4(IntA4,IntA5,IntT8,IntT9); PUSH4(IntT10,IntT11,IntT12,IntAt); sp=(ptr_t)context.IntSp; #elif defined(CPPCHECK) sp=(ptr_t)(word)cnt; #else #error Architecture is not supported #endif #undef context GC_ASSERT(cnt==PUSHED_REGS_COUNT); return sp; } STATIC word GC_push_stack_for(GC_thread thread,DWORD me) { ptr_t sp,stack_min; struct GC_traced_stack_sect_s*traced_stack_sect= thread->traced_stack_sect; if (thread->id==me){ GC_ASSERT(thread->thread_blocked_sp==NULL); sp=GC_approx_sp(); } else if ((sp=thread->thread_blocked_sp)==NULL){ #ifdef RETRY_GET_THREAD_CONTEXT word*regs=thread->context_regs; if (thread->suspended){ sp=thread->context_sp; } else #else word regs[PUSHED_REGS_COUNT]; #endif { CONTEXT context; context.ContextFlags=GET_THREAD_CONTEXT_FLAGS; if (GetThreadContext(THREAD_HANDLE(thread),&context)){ sp=copy_ptr_regs(regs,&context); } else { #ifdef RETRY_GET_THREAD_CONTEXT sp=thread->context_sp; if (NULL==sp){ return 0; } #else ABORT("GetThreadContext failed"); #endif } } #ifdef THREAD_LOCAL_ALLOC GC_ASSERT(thread->suspended||!GC_world_stopped); #endif #ifndef WOW64_THREAD_CONTEXT_WORKAROUND GC_push_many_regs(regs,PUSHED_REGS_COUNT); #else GC_push_many_regs(regs+2,PUSHED_REGS_COUNT - 2); if (isWow64){ DWORD ContextFlags=(DWORD)regs[0]; WORD SegFs=(WORD)regs[1]; if ((ContextFlags&CONTEXT_EXCEPTION_REPORTING)!=0 &&(ContextFlags&(CONTEXT_EXCEPTION_ACTIVE ))!=0){ LDT_ENTRY selector; PNT_TIB tib; if (!GetThreadSelectorEntry(THREAD_HANDLE(thread),SegFs,&selector)) ABORT("GetThreadSelectorEntry failed"); tib=(PNT_TIB)(selector.BaseLow |(selector.HighWord.Bits.BaseMid<<16) |(selector.HighWord.Bits.BaseHi<<24)); #ifdef DEBUG_THREADS GC_log_printf("TIB stack limit/base:%p .. %p\n", (void*)tib->StackLimit,(void*)tib->StackBase); #endif GC_ASSERT(!((word)thread->stack_base COOLER_THAN (word)tib->StackBase)); if (thread->stack_base!=thread->initial_stack_base &&((word)thread->stack_base<=(word)tib->StackLimit ||(word)tib->StackBase < (word)thread->stack_base)){ WARN("GetThreadContext might return stale register values" " including ESP=%p\n",sp); } else { sp=(ptr_t)tib->StackLimit; } } #ifdef DEBUG_THREADS else { static GC_bool logged; if (!logged &&(ContextFlags&CONTEXT_EXCEPTION_REPORTING)==0){ GC_log_printf("CONTEXT_EXCEPTION_REQUEST not supported\n"); logged=TRUE; } } #endif } #endif } if (thread->last_stack_min==ADDR_LIMIT){ #ifdef MSWINCE if (GC_dont_query_stack_min){ stack_min=GC_wince_evaluate_stack_min(traced_stack_sect!=NULL? (ptr_t)traced_stack_sect:thread->stack_base); } else #endif { stack_min=GC_get_stack_min(traced_stack_sect!=NULL? (ptr_t)traced_stack_sect:thread->stack_base); UNPROTECT_THREAD(thread); thread->last_stack_min=stack_min; } } else { if (traced_stack_sect!=NULL&& (word)thread->last_stack_min > (word)traced_stack_sect){ UNPROTECT_THREAD(thread); thread->last_stack_min=(ptr_t)traced_stack_sect; } if ((word)sp < (word)thread->stack_base &&(word)sp>=(word)thread->last_stack_min){ stack_min=sp; } else { if (may_be_in_stack(thread->id==me&& (word)sp < (word)thread->last_stack_min? sp:thread->last_stack_min)){ stack_min=(ptr_t)last_info.BaseAddress; if ((word)sp < (word)stack_min ||(word)sp>=(word)thread->stack_base) stack_min=GC_get_stack_min(thread->last_stack_min); } else { stack_min=GC_get_stack_min(thread->stack_base); } UNPROTECT_THREAD(thread); thread->last_stack_min=stack_min; } } GC_ASSERT(GC_dont_query_stack_min ||stack_min==GC_get_stack_min(thread->stack_base) ||((word)sp>=(word)stack_min &&(word)stack_min < (word)thread->stack_base &&(word)stack_min > (word)GC_get_stack_min(thread->stack_base))); if ((word)sp>=(word)stack_min&&(word)sp < (word)thread->stack_base){ #ifdef DEBUG_THREADS GC_log_printf("Pushing stack for 0x%x from sp %p to %p from 0x%x\n", (int)thread->id,(void*)sp,(void*)thread->stack_base, (int)me); #endif GC_push_all_stack_sections(sp,thread->stack_base,traced_stack_sect); } else { if (thread->id==me||(word)sp>=(word)thread->stack_base ||(word)(sp+GC_page_size)< (word)stack_min) WARN("Thread stack pointer %p out of range,pushing everything\n", sp); #ifdef DEBUG_THREADS GC_log_printf("Pushing stack for 0x%x from (min)%p to %p from 0x%x\n", (int)thread->id,(void*)stack_min, (void*)thread->stack_base,(int)me); #endif GC_push_all_stack(stack_min,thread->stack_base); } return thread->stack_base - sp; } GC_INNER void GC_push_all_stacks(void) { DWORD thread_id=GetCurrentThreadId(); GC_bool found_me=FALSE; #ifndef SMALL_CONFIG unsigned nthreads=0; #endif word total_size=0; #ifndef GC_NO_THREADS_DISCOVERY if (GC_win32_dll_threads){ int i; LONG my_max=GC_get_max_thread_index(); for (i=0;i<=my_max;i++){ GC_thread t=(GC_thread)(dll_thread_table+i); if (t->tm.in_use&&t->stack_base){ #ifndef SMALL_CONFIG ++nthreads; #endif total_size+=GC_push_stack_for(t,thread_id); if (t->id==thread_id)found_me=TRUE; } } } else #endif { int i; for (i=0;i < THREAD_TABLE_SZ;i++){ GC_thread t; for (t=GC_threads[i];t!=0;t=t->tm.next){ if (!KNOWN_FINISHED(t)&&t->stack_base){ #ifndef SMALL_CONFIG ++nthreads; #endif total_size+=GC_push_stack_for(t,thread_id); if (t->id==thread_id)found_me=TRUE; } } } } #ifndef SMALL_CONFIG GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks%s\n",nthreads, GC_win32_dll_threads? " based on DllMain thread tracking":""); #endif if (!found_me&&!GC_in_thread_creation) ABORT("Collecting from unknown thread"); GC_total_stacksize=total_size; } #ifdef PARALLEL_MARK #ifndef MAX_MARKERS #define MAX_MARKERS 16 #endif static ptr_t marker_sp[MAX_MARKERS - 1]; #ifdef IA64 static ptr_t marker_bsp[MAX_MARKERS - 1]; #endif static ptr_t marker_last_stack_min[MAX_MARKERS - 1]; #endif GC_INNER void GC_get_next_stack(char*start,char*limit, char**lo,char**hi) { int i; char*current_min=ADDR_LIMIT; ptr_t*plast_stack_min=NULL; GC_thread thread=NULL; if (GC_win32_dll_threads){ LONG my_max=GC_get_max_thread_index(); for (i=0;i<=my_max;i++){ ptr_t s=(ptr_t)(dll_thread_table[i].stack_base); if ((word)s > (word)start&&(word)s < (word)current_min){ plast_stack_min=(ptr_t*) &dll_thread_table[i].last_stack_min; current_min=s; #if defined(CPPCHECK) thread=(GC_thread)&dll_thread_table[i]; #endif } } } else { for (i=0;i < THREAD_TABLE_SZ;i++){ GC_thread t; for (t=GC_threads[i];t!=0;t=t->tm.next){ ptr_t s=t->stack_base; if ((word)s > (word)start&&(word)s < (word)current_min){ plast_stack_min=&t->last_stack_min; thread=t; current_min=s; } } } #ifdef PARALLEL_MARK for (i=0;i < GC_markers_m1;++i){ ptr_t s=marker_sp[i]; #ifdef IA64 #endif if ((word)s > (word)start&&(word)s < (word)current_min){ GC_ASSERT(marker_last_stack_min[i]!=NULL); plast_stack_min=&marker_last_stack_min[i]; current_min=s; thread=NULL; } } #endif } *hi=current_min; if (current_min==ADDR_LIMIT){ *lo=ADDR_LIMIT; return; } GC_ASSERT((word)current_min > (word)start&&plast_stack_min!=NULL); #ifdef MSWINCE if (GC_dont_query_stack_min){ *lo=GC_wince_evaluate_stack_min(current_min); return; } #endif if ((word)current_min > (word)limit&&!may_be_in_stack(limit)){ *lo=ADDR_LIMIT; return; } if (*plast_stack_min==ADDR_LIMIT ||!may_be_in_stack(*plast_stack_min)){ *lo=GC_get_stack_min(current_min); } else { *lo=GC_get_stack_min(*plast_stack_min); } if (thread!=NULL){ UNPROTECT_THREAD(thread); } *plast_stack_min=*lo; } #ifdef PARALLEL_MARK #if defined(GC_PTHREADS)&&!defined(GC_PTHREADS_PARAMARK) #if!defined(__MINGW32__) #define GC_PTHREADS_PARAMARK #endif #endif #if!defined(GC_PTHREADS_PARAMARK) STATIC HANDLE GC_marker_cv[MAX_MARKERS - 1]={0}; STATIC DWORD GC_marker_Id[MAX_MARKERS - 1]={0}; #endif #if defined(GC_PTHREADS)&&defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) static void set_marker_thread_name(unsigned id) { char name_buf[16]; int len=sizeof("GC-marker-")- 1; BCOPY("GC-marker-",name_buf,len); if (id>=10) name_buf[len++]=(char)('0'+(id/10)% 10); name_buf[len]=(char)('0'+id % 10); name_buf[len+1]='\0'; if (pthread_setname_np(pthread_self(),name_buf)!=0) WARN("pthread_setname_np failed\n",0); } #elif!defined(MSWINCE) static FARPROC setThreadDescription_fn; static void set_marker_thread_name(unsigned id) { WCHAR name_buf[16]; int len=sizeof(L"GC-marker-")/sizeof(WCHAR)- 1; HRESULT hr; if (!setThreadDescription_fn)return; BCOPY(L"GC-marker-",name_buf,len*sizeof(WCHAR)); if (id>=10) name_buf[len++]=(WCHAR)('0'+(id/10)% 10); name_buf[len]=(WCHAR)('0'+id % 10); name_buf[len+1]=0; hr=(*(HRESULT (WINAPI*)(HANDLE,const WCHAR*)) (word)setThreadDescription_fn)(GetCurrentThread(),name_buf); if (FAILED(hr)) WARN("SetThreadDescription failed\n",0); } #else #define set_marker_thread_name(id)(void)(id) #endif #ifdef GC_PTHREADS_PARAMARK STATIC void*GC_mark_thread(void*id) #elif defined(MSWINCE) STATIC DWORD WINAPI GC_mark_thread(LPVOID id) #else STATIC unsigned __stdcall GC_mark_thread(void*id) #endif { word my_mark_no=0; if ((word)id==GC_WORD_MAX)return 0; set_marker_thread_name((unsigned)(word)id); marker_sp[(word)id]=GC_approx_sp(); #ifdef IA64 marker_bsp[(word)id]=GC_save_regs_in_stack(); #endif #if!defined(GC_PTHREADS_PARAMARK) GC_marker_Id[(word)id]=GetCurrentThreadId(); #endif GC_acquire_mark_lock(); if (0==--GC_fl_builder_count) GC_notify_all_builder(); for (;;++my_mark_no){ if (my_mark_no - GC_mark_no > (word)2){ my_mark_no=GC_mark_no; } #ifdef DEBUG_THREADS GC_log_printf("Starting mark helper for mark number %lu\n", (unsigned long)my_mark_no); #endif GC_help_marker(my_mark_no); } } #ifndef GC_ASSERTIONS #define SET_MARK_LOCK_HOLDER (void)0 #define UNSET_MARK_LOCK_HOLDER (void)0 #endif #ifdef CAN_HANDLE_FORK static int available_markers_m1=0; #else #define available_markers_m1 GC_markers_m1 #endif #ifdef GC_PTHREADS_PARAMARK #include #if defined(GC_ASSERTIONS)&&!defined(NUMERIC_THREAD_ID) #define NUMERIC_THREAD_ID(id)(unsigned long)(word)GC_PTHREAD_PTRVAL(id) #endif #ifdef CAN_HANDLE_FORK static pthread_cond_t mark_cv; #else static pthread_cond_t mark_cv=PTHREAD_COND_INITIALIZER; #endif GC_INNER void GC_start_mark_threads_inner(void) { int i; pthread_attr_t attr; pthread_t new_thread; #ifndef NO_MARKER_SPECIAL_SIGMASK sigset_t set,oldset; #endif GC_ASSERT(I_DONT_HOLD_LOCK()); if (available_markers_m1<=0)return; #ifdef CAN_HANDLE_FORK if (GC_parallel)return; { pthread_cond_t mark_cv_local=PTHREAD_COND_INITIALIZER; BCOPY(&mark_cv_local,&mark_cv,sizeof(mark_cv)); } #endif GC_ASSERT(GC_fl_builder_count==0); if (0!=pthread_attr_init(&attr))ABORT("pthread_attr_init failed"); if (0!=pthread_attr_setdetachstate(&attr,PTHREAD_CREATE_DETACHED)) ABORT("pthread_attr_setdetachstate failed"); #ifndef NO_MARKER_SPECIAL_SIGMASK if (sigfillset(&set)!=0) ABORT("sigfillset failed"); if (pthread_sigmask(SIG_BLOCK,&set,&oldset)< 0){ WARN("pthread_sigmask set failed,no markers started," " errno=%" WARN_PRIdPTR "\n",errno); GC_markers_m1=0; (void)pthread_attr_destroy(&attr); return; } #endif #ifdef CAN_HANDLE_FORK GC_markers_m1=available_markers_m1; #endif for (i=0;i < available_markers_m1;++i){ marker_last_stack_min[i]=ADDR_LIMIT; if (0!=pthread_create(&new_thread,&attr, GC_mark_thread,(void*)(word)i)){ WARN("Marker thread creation failed\n",0); GC_markers_m1=i; break; } } #ifndef NO_MARKER_SPECIAL_SIGMASK if (pthread_sigmask(SIG_SETMASK,&oldset,NULL)< 0){ WARN("pthread_sigmask restore failed,errno=%" WARN_PRIdPTR "\n", errno); } #endif (void)pthread_attr_destroy(&attr); GC_wait_for_markers_init(); GC_COND_LOG_PRINTF("Started %d mark helper threads\n",GC_markers_m1); } #ifdef GC_ASSERTIONS STATIC unsigned long GC_mark_lock_holder=NO_THREAD; #define SET_MARK_LOCK_HOLDER (void)(GC_mark_lock_holder=NUMERIC_THREAD_ID(pthread_self())) #define UNSET_MARK_LOCK_HOLDER do { GC_ASSERT(GC_mark_lock_holder==NUMERIC_THREAD_ID(pthread_self()));GC_mark_lock_holder=NO_THREAD;} while (0) #endif static pthread_mutex_t mark_mutex=PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t builder_cv=PTHREAD_COND_INITIALIZER; #ifdef LOCK_STATS volatile AO_t GC_block_count=0; #endif GC_INNER void GC_acquire_mark_lock(void) { #if defined(NUMERIC_THREAD_ID_UNIQUE)&&!defined(THREAD_SANITIZER) GC_ASSERT(GC_mark_lock_holder!=NUMERIC_THREAD_ID(pthread_self())); #endif if (pthread_mutex_lock(&mark_mutex)!=0){ ABORT("pthread_mutex_lock failed"); } #ifdef LOCK_STATS (void)AO_fetch_and_add1(&GC_block_count); #endif SET_MARK_LOCK_HOLDER; } GC_INNER void GC_release_mark_lock(void) { UNSET_MARK_LOCK_HOLDER; if (pthread_mutex_unlock(&mark_mutex)!=0){ ABORT("pthread_mutex_unlock failed"); } } STATIC void GC_wait_builder(void) { UNSET_MARK_LOCK_HOLDER; if (pthread_cond_wait(&builder_cv,&mark_mutex)!=0){ ABORT("pthread_cond_wait failed"); } GC_ASSERT(GC_mark_lock_holder==NO_THREAD); SET_MARK_LOCK_HOLDER; } GC_INNER void GC_wait_for_reclaim(void) { GC_acquire_mark_lock(); while (GC_fl_builder_count > 0){ GC_wait_builder(); } GC_release_mark_lock(); } GC_INNER void GC_notify_all_builder(void) { GC_ASSERT(GC_mark_lock_holder==NUMERIC_THREAD_ID(pthread_self())); if (pthread_cond_broadcast(&builder_cv)!=0){ ABORT("pthread_cond_broadcast failed"); } } GC_INNER void GC_wait_marker(void) { GC_ASSERT(GC_parallel); UNSET_MARK_LOCK_HOLDER; if (pthread_cond_wait(&mark_cv,&mark_mutex)!=0){ ABORT("pthread_cond_wait failed"); } GC_ASSERT(GC_mark_lock_holder==NO_THREAD); SET_MARK_LOCK_HOLDER; } GC_INNER void GC_notify_all_marker(void) { GC_ASSERT(GC_parallel); if (pthread_cond_broadcast(&mark_cv)!=0){ ABORT("pthread_cond_broadcast failed"); } } #else #ifndef MARK_THREAD_STACK_SIZE #define MARK_THREAD_STACK_SIZE 0 #endif static HANDLE mark_mutex_event=(HANDLE)0; static HANDLE builder_cv=(HANDLE)0; static HANDLE mark_cv=(HANDLE)0; GC_INNER void GC_start_mark_threads_inner(void) { int i; GC_ASSERT(I_DONT_HOLD_LOCK()); if (available_markers_m1<=0)return; GC_ASSERT(GC_fl_builder_count==0); for (i=0;i < GC_markers_m1;++i){ if ((GC_marker_cv[i]=CreateEvent(NULL, TRUE, FALSE, NULL))==(HANDLE)0) ABORT("CreateEvent failed"); } for (i=0;i < GC_markers_m1;++i){ #if defined(MSWINCE)||defined(MSWIN_XBOX1) HANDLE handle; DWORD thread_id; marker_last_stack_min[i]=ADDR_LIMIT; handle=CreateThread(NULL, MARK_THREAD_STACK_SIZE, GC_mark_thread,(LPVOID)(word)i, 0,&thread_id); if (handle==NULL){ WARN("Marker thread creation failed\n",0); break; } else { CloseHandle(handle); } #else GC_uintptr_t handle; unsigned thread_id; marker_last_stack_min[i]=ADDR_LIMIT; handle=_beginthreadex(NULL, MARK_THREAD_STACK_SIZE,GC_mark_thread, (void*)(word)i,0,&thread_id); if (!handle||handle==(GC_uintptr_t)-1L){ WARN("Marker thread creation failed\n",0); break; } else { } #endif } while (GC_markers_m1 > i){ GC_markers_m1--; CloseHandle(GC_marker_cv[GC_markers_m1]); } GC_wait_for_markers_init(); GC_COND_LOG_PRINTF("Started %d mark helper threads\n",GC_markers_m1); if (i==0){ CloseHandle(mark_cv); CloseHandle(builder_cv); CloseHandle(mark_mutex_event); } } #ifdef GC_ASSERTIONS STATIC DWORD GC_mark_lock_holder=NO_THREAD; #define SET_MARK_LOCK_HOLDER (void)(GC_mark_lock_holder=GetCurrentThreadId()) #define UNSET_MARK_LOCK_HOLDER do { GC_ASSERT(GC_mark_lock_holder==GetCurrentThreadId());GC_mark_lock_holder=NO_THREAD;} while (0) #endif STATIC LONG GC_mark_mutex_state=0; #ifdef LOCK_STATS volatile AO_t GC_block_count=0; volatile AO_t GC_unlocked_count=0; #endif GC_INNER void GC_acquire_mark_lock(void) { #ifndef THREAD_SANITIZER GC_ASSERT(GC_mark_lock_holder!=GetCurrentThreadId()); #endif if (InterlockedExchange(&GC_mark_mutex_state,1)!=0){ #ifdef LOCK_STATS (void)AO_fetch_and_add1(&GC_block_count); #endif while (InterlockedExchange(&GC_mark_mutex_state, -1)!=0){ if (WaitForSingleObject(mark_mutex_event,INFINITE)==WAIT_FAILED) ABORT("WaitForSingleObject failed"); } } #ifdef LOCK_STATS else { (void)AO_fetch_and_add1(&GC_unlocked_count); } #endif GC_ASSERT(GC_mark_lock_holder==NO_THREAD); SET_MARK_LOCK_HOLDER; } GC_INNER void GC_release_mark_lock(void) { UNSET_MARK_LOCK_HOLDER; if (InterlockedExchange(&GC_mark_mutex_state,0)< 0){ if (SetEvent(mark_mutex_event)==FALSE) ABORT("SetEvent failed"); } } GC_INNER void GC_wait_for_reclaim(void) { GC_ASSERT(builder_cv!=0); for (;;){ GC_acquire_mark_lock(); if (GC_fl_builder_count==0) break; if (ResetEvent(builder_cv)==FALSE) ABORT("ResetEvent failed"); GC_release_mark_lock(); if (WaitForSingleObject(builder_cv,INFINITE)==WAIT_FAILED) ABORT("WaitForSingleObject failed"); } GC_release_mark_lock(); } GC_INNER void GC_notify_all_builder(void) { GC_ASSERT(GC_mark_lock_holder==GetCurrentThreadId()); GC_ASSERT(builder_cv!=0); GC_ASSERT(GC_fl_builder_count==0); if (SetEvent(builder_cv)==FALSE) ABORT("SetEvent failed"); } GC_INNER void GC_wait_marker(void) { HANDLE event=mark_cv; DWORD thread_id=GetCurrentThreadId(); int i=GC_markers_m1; while (i--> 0){ if (GC_marker_Id[i]==thread_id){ event=GC_marker_cv[i]; break; } } if (ResetEvent(event)==FALSE) ABORT("ResetEvent failed"); GC_release_mark_lock(); if (WaitForSingleObject(event,INFINITE)==WAIT_FAILED) ABORT("WaitForSingleObject failed"); GC_acquire_mark_lock(); } GC_INNER void GC_notify_all_marker(void) { DWORD thread_id=GetCurrentThreadId(); int i=GC_markers_m1; while (i--> 0){ if (SetEvent(GC_marker_Id[i]!=thread_id?GC_marker_cv[i]: mark_cv)==FALSE) ABORT("SetEvent failed"); } } #endif static unsigned required_markers_cnt=0; #endif typedef struct { LPTHREAD_START_ROUTINE start; LPVOID param; } thread_args; STATIC void*GC_CALLBACK GC_win32_start_inner(struct GC_stack_base*sb, void*arg) { void*ret; LPTHREAD_START_ROUTINE start=((thread_args*)arg)->start; LPVOID param=((thread_args*)arg)->param; GC_register_my_thread(sb); #ifdef DEBUG_THREADS GC_log_printf("thread 0x%lx starting...\n",(long)GetCurrentThreadId()); #endif GC_free(arg); #if!defined(__GNUC__)&&!defined(NO_CRT) ret=NULL; __try #endif { ret=(void*)(word)(*start)(param); } #if!defined(__GNUC__)&&!defined(NO_CRT) __finally #endif { GC_unregister_my_thread(); } #ifdef DEBUG_THREADS GC_log_printf("thread 0x%lx returned from start routine\n", (long)GetCurrentThreadId()); #endif return ret; } STATIC DWORD WINAPI GC_win32_start(LPVOID arg) { return (DWORD)(word)GC_call_with_stack_base(GC_win32_start_inner,arg); } GC_API HANDLE WINAPI GC_CreateThread( LPSECURITY_ATTRIBUTES lpThreadAttributes, GC_WIN32_SIZE_T dwStackSize, LPTHREAD_START_ROUTINE lpStartAddress, LPVOID lpParameter,DWORD dwCreationFlags, LPDWORD lpThreadId) { if (!EXPECT(parallel_initialized,TRUE)) GC_init_parallel(); #ifdef DEBUG_THREADS GC_log_printf("About to create a thread from 0x%lx\n", (long)GetCurrentThreadId()); #endif if (GC_win32_dll_threads){ return CreateThread(lpThreadAttributes,dwStackSize,lpStartAddress, lpParameter,dwCreationFlags,lpThreadId); } else { thread_args*args= (thread_args*)GC_malloc_uncollectable(sizeof(thread_args)); HANDLE thread_h; if (NULL==args){ SetLastError(ERROR_NOT_ENOUGH_MEMORY); return NULL; } args->start=lpStartAddress; args->param=lpParameter; GC_dirty(args); REACHABLE_AFTER_DIRTY(lpParameter); set_need_to_lock(); thread_h=CreateThread(lpThreadAttributes,dwStackSize,GC_win32_start, args,dwCreationFlags,lpThreadId); if (thread_h==0)GC_free(args); return thread_h; } } GC_API DECLSPEC_NORETURN void WINAPI GC_ExitThread(DWORD dwExitCode) { GC_unregister_my_thread(); ExitThread(dwExitCode); } #if!defined(CYGWIN32)&&!defined(MSWINCE)&&!defined(MSWIN_XBOX1)&&!defined(NO_CRT) GC_API GC_uintptr_t GC_CALL GC_beginthreadex( void*security,unsigned stack_size, unsigned (__stdcall*start_address)(void*), void*arglist,unsigned initflag, unsigned*thrdaddr) { if (!EXPECT(parallel_initialized,TRUE)) GC_init_parallel(); #ifdef DEBUG_THREADS GC_log_printf("About to create a thread from 0x%lx\n", (long)GetCurrentThreadId()); #endif if (GC_win32_dll_threads){ return _beginthreadex(security,stack_size,start_address, arglist,initflag,thrdaddr); } else { GC_uintptr_t thread_h; thread_args*args= (thread_args*)GC_malloc_uncollectable(sizeof(thread_args)); if (NULL==args){ errno=EAGAIN; return 0; } args->start=(LPTHREAD_START_ROUTINE)start_address; args->param=arglist; GC_dirty(args); REACHABLE_AFTER_DIRTY(arglist); set_need_to_lock(); thread_h=_beginthreadex(security,stack_size, (unsigned (__stdcall*)(void*))GC_win32_start, args,initflag,thrdaddr); if (thread_h==0)GC_free(args); return thread_h; } } GC_API void GC_CALL GC_endthreadex(unsigned retval) { GC_unregister_my_thread(); _endthreadex(retval); } #endif #ifdef GC_WINMAIN_REDIRECT #if defined(MSWINCE)&&defined(UNDER_CE) #define WINMAIN_LPTSTR LPWSTR #else #define WINMAIN_LPTSTR LPSTR #endif #undef WinMain int WINAPI GC_WinMain(HINSTANCE,HINSTANCE,WINMAIN_LPTSTR,int); typedef struct { HINSTANCE hInstance; HINSTANCE hPrevInstance; WINMAIN_LPTSTR lpCmdLine; int nShowCmd; } main_thread_args; static DWORD WINAPI main_thread_start(LPVOID arg) { main_thread_args*args=(main_thread_args*)arg; return (DWORD)GC_WinMain(args->hInstance,args->hPrevInstance, args->lpCmdLine,args->nShowCmd); } STATIC void*GC_waitForSingleObjectInfinite(void*handle) { return (void*)(word)WaitForSingleObject((HANDLE)handle,INFINITE); } #ifndef WINMAIN_THREAD_STACK_SIZE #define WINMAIN_THREAD_STACK_SIZE 0 #endif int WINAPI WinMain(HINSTANCE hInstance,HINSTANCE hPrevInstance, WINMAIN_LPTSTR lpCmdLine,int nShowCmd) { DWORD exit_code=1; main_thread_args args={ hInstance,hPrevInstance,lpCmdLine,nShowCmd }; HANDLE thread_h; DWORD thread_id; GC_INIT(); thread_h=GC_CreateThread(NULL, WINMAIN_THREAD_STACK_SIZE, main_thread_start,&args,0, &thread_id); if (thread_h!=NULL){ if ((DWORD)(word)GC_do_blocking(GC_waitForSingleObjectInfinite, (void*)thread_h)==WAIT_FAILED) ABORT("WaitForSingleObject(main_thread)failed"); GetExitCodeThread (thread_h,&exit_code); CloseHandle (thread_h); } else { ABORT("GC_CreateThread(main_thread)failed"); } #ifdef MSWINCE GC_deinit(); #endif return (int)exit_code; } #endif GC_API void GC_CALL GC_set_markers_count(unsigned markers GC_ATTR_UNUSED) { #ifdef PARALLEL_MARK required_markers_cnt=markers < MAX_MARKERS?markers:MAX_MARKERS; #endif } GC_INNER void GC_thr_init(void) { struct GC_stack_base sb; #if (!defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID)&&!defined(MSWINCE)&&defined(PARALLEL_MARK))||defined(WOW64_THREAD_CONTEXT_WORKAROUND) HMODULE hK32=GetModuleHandle(TEXT("kernel32.dll")); #endif GC_ASSERT(I_HOLD_LOCK()); if (GC_thr_initialized)return; GC_ASSERT((word)&GC_threads % sizeof(word)==0); #ifdef GC_NO_THREADS_DISCOVERY #define GC_main_thread GetCurrentThreadId() #else GC_main_thread=GetCurrentThreadId(); #endif GC_thr_initialized=TRUE; #ifdef CAN_HANDLE_FORK if (GC_handle_fork){ #ifdef CAN_CALL_ATFORK if (pthread_atfork(fork_prepare_proc,fork_parent_proc, fork_child_proc)==0){ GC_handle_fork=1; } else #endif if (GC_handle_fork!=-1) ABORT("pthread_atfork failed"); } #endif #ifdef WOW64_THREAD_CONTEXT_WORKAROUND if (hK32){ FARPROC pfn=GetProcAddress(hK32,"IsWow64Process"); if (pfn &&!(*(BOOL (WINAPI*)(HANDLE,BOOL*))(word)pfn)( GetCurrentProcess(),&isWow64)) isWow64=FALSE; } #endif sb.mem_base=GC_stackbottom; GC_ASSERT(sb.mem_base!=NULL); #ifdef IA64 sb.reg_base=GC_register_stackbottom; #endif #if defined(PARALLEL_MARK) { char*markers_string=GETENV("GC_MARKERS"); int markers=required_markers_cnt; if (markers_string!=NULL){ markers=atoi(markers_string); if (markers<=0||markers > MAX_MARKERS){ WARN("Too big or invalid number of mark threads:%" WARN_PRIdPTR ";using maximum threads\n",(signed_word)markers); markers=MAX_MARKERS; } } else if (0==markers){ #ifdef MSWINCE markers=(int)GC_sysinfo.dwNumberOfProcessors; #else #ifdef _WIN64 DWORD_PTR procMask=0; DWORD_PTR sysMask; #else DWORD procMask=0; DWORD sysMask; #endif int ncpu=0; if ( #ifdef __cplusplus GetProcessAffinityMask(GetCurrentProcess(),&procMask,&sysMask) #else GetProcessAffinityMask(GetCurrentProcess(), (void*)&procMask,(void*)&sysMask) #endif &&procMask){ do { ncpu++; } while ((procMask&=procMask - 1)!=0); } markers=ncpu; #endif #if defined(GC_MIN_MARKERS)&&!defined(CPPCHECK) if (markers < GC_MIN_MARKERS) markers=GC_MIN_MARKERS; #endif if (markers > MAX_MARKERS) markers=MAX_MARKERS; } available_markers_m1=markers - 1; } if (GC_win32_dll_threads||available_markers_m1<=0){ GC_parallel=FALSE; GC_COND_LOG_PRINTF( "Single marker thread,turning off parallel marking\n"); } else { #ifndef GC_PTHREADS_PARAMARK mark_mutex_event=CreateEvent(NULL, FALSE, FALSE,NULL); builder_cv=CreateEvent(NULL, TRUE, FALSE,NULL); mark_cv=CreateEvent(NULL,TRUE, FALSE,NULL); if (mark_mutex_event==(HANDLE)0||builder_cv==(HANDLE)0 ||mark_cv==(HANDLE)0) ABORT("CreateEvent failed"); #endif #if!defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID)&&!defined(MSWINCE) if (hK32) setThreadDescription_fn=GetProcAddress(hK32, "SetThreadDescription"); #endif } #endif GC_ASSERT(0==GC_lookup_thread_inner(GC_main_thread)); GC_register_my_thread_inner(&sb,GC_main_thread); #undef GC_main_thread } #ifdef GC_PTHREADS struct start_info { void*(*start_routine)(void*); void*arg; GC_bool detached; }; GC_API int GC_pthread_join(pthread_t pthread_id,void**retval) { int result; #ifndef GC_WIN32_PTHREADS GC_thread t; #endif DCL_LOCK_STATE; GC_ASSERT(!GC_win32_dll_threads); #ifdef DEBUG_THREADS GC_log_printf("thread %p(0x%lx)is joining thread %p\n", (void*)GC_PTHREAD_PTRVAL(pthread_self()), (long)GetCurrentThreadId(), (void*)GC_PTHREAD_PTRVAL(pthread_id)); #endif #ifndef GC_WIN32_PTHREADS while ((t=GC_lookup_pthread(pthread_id))==0) Sleep(10); #endif result=pthread_join(pthread_id,retval); if (0==result){ #ifdef GC_WIN32_PTHREADS GC_thread t=GC_lookup_pthread(pthread_id); if (NULL==t)ABORT("Thread not registered"); #endif LOCK(); if ((t->flags&FINISHED)!=0){ GC_delete_gc_thread_no_free(t); GC_INTERNAL_FREE(t); } UNLOCK(); } #ifdef DEBUG_THREADS GC_log_printf("thread %p(0x%lx)join with thread %p %s\n", (void*)GC_PTHREAD_PTRVAL(pthread_self()), (long)GetCurrentThreadId(), (void*)GC_PTHREAD_PTRVAL(pthread_id), result!=0?"failed":"succeeded"); #endif return result; } GC_API int GC_pthread_create(pthread_t*new_thread, GC_PTHREAD_CREATE_CONST pthread_attr_t*attr, void*(*start_routine)(void*),void*arg) { int result; struct start_info*si; if (!EXPECT(parallel_initialized,TRUE)) GC_init_parallel(); GC_ASSERT(!GC_win32_dll_threads); si=(struct start_info*)GC_malloc_uncollectable( sizeof(struct start_info)); if (NULL==si) return EAGAIN; si->start_routine=start_routine; si->arg=arg; GC_dirty(si); REACHABLE_AFTER_DIRTY(arg); if (attr!=0&& pthread_attr_getdetachstate(attr,&si->detached) ==PTHREAD_CREATE_DETACHED){ si->detached=TRUE; } #ifdef DEBUG_THREADS GC_log_printf("About to create a thread from %p(0x%lx)\n", (void*)GC_PTHREAD_PTRVAL(pthread_self()), (long)GetCurrentThreadId()); #endif set_need_to_lock(); result=pthread_create(new_thread,attr,GC_pthread_start,si); if (result){ GC_free(si); } return(result); } STATIC void*GC_CALLBACK GC_pthread_start_inner(struct GC_stack_base*sb, void*arg) { struct start_info*si=(struct start_info*)arg; void*result; void*(*start)(void*); void*start_arg; DWORD thread_id=GetCurrentThreadId(); pthread_t pthread_id=pthread_self(); GC_thread me; DCL_LOCK_STATE; #ifdef DEBUG_THREADS GC_log_printf("thread %p(0x%x)starting...\n", (void*)GC_PTHREAD_PTRVAL(pthread_id),(int)thread_id); #endif GC_ASSERT(!GC_win32_dll_threads); LOCK(); me=GC_register_my_thread_inner(sb,thread_id); SET_PTHREAD_MAP_CACHE(pthread_id,thread_id); GC_ASSERT(me!=&first_thread); me->pthread_id=pthread_id; if (si->detached)me->flags|=DETACHED; UNLOCK(); start=si->start_routine; start_arg=si->arg; GC_free(si); pthread_cleanup_push(GC_thread_exit_proc,(void*)me); result=(*start)(start_arg); me->status=result; GC_dirty(me); pthread_cleanup_pop(1); #ifdef DEBUG_THREADS GC_log_printf("thread %p(0x%x)returned from start routine\n", (void*)GC_PTHREAD_PTRVAL(pthread_id),(int)thread_id); #endif return(result); } STATIC void*GC_pthread_start(void*arg) { return GC_call_with_stack_base(GC_pthread_start_inner,arg); } STATIC void GC_thread_exit_proc(void*arg) { GC_thread me=(GC_thread)arg; DCL_LOCK_STATE; GC_ASSERT(!GC_win32_dll_threads); #ifdef DEBUG_THREADS GC_log_printf("thread %p(0x%lx)called pthread_exit()\n", (void*)GC_PTHREAD_PTRVAL(pthread_self()), (long)GetCurrentThreadId()); #endif LOCK(); GC_wait_for_gc_completion(FALSE); #if defined(THREAD_LOCAL_ALLOC) GC_ASSERT(GC_getspecific(GC_thread_key)==&me->tlfs); GC_destroy_thread_local(&(me->tlfs)); #endif if (me->flags&DETACHED){ GC_delete_thread(GetCurrentThreadId()); } else { me->flags|=FINISHED; } #if defined(THREAD_LOCAL_ALLOC) GC_remove_specific(GC_thread_key); #endif UNLOCK(); } #ifndef GC_NO_PTHREAD_SIGMASK GC_API int GC_pthread_sigmask(int how,const sigset_t*set, sigset_t*oset) { return pthread_sigmask(how,set,oset); } #endif GC_API int GC_pthread_detach(pthread_t thread) { int result; GC_thread t; DCL_LOCK_STATE; GC_ASSERT(!GC_win32_dll_threads); while ((t=GC_lookup_pthread(thread))==NULL) Sleep(10); result=pthread_detach(thread); if (result==0){ LOCK(); t->flags|=DETACHED; if ((t->flags&FINISHED)!=0){ GC_delete_gc_thread_no_free(t); GC_INTERNAL_FREE(t); } UNLOCK(); } return result; } #elif!defined(GC_NO_THREADS_DISCOVERY) #ifdef GC_INSIDE_DLL GC_API #else #define GC_DllMain DllMain #endif BOOL WINAPI GC_DllMain(HINSTANCE inst GC_ATTR_UNUSED,ULONG reason, LPVOID reserved GC_ATTR_UNUSED) { DWORD thread_id; if (!GC_win32_dll_threads&¶llel_initialized)return TRUE; switch (reason){ case DLL_THREAD_ATTACH: #ifdef PARALLEL_MARK if (GC_parallel){ break; } #endif case DLL_PROCESS_ATTACH: thread_id=GetCurrentThreadId(); if (parallel_initialized&&GC_main_thread!=thread_id){ #ifdef PARALLEL_MARK ABORT("Cannot initialize parallel marker from DllMain"); #else struct GC_stack_base sb; #ifdef GC_ASSERTIONS int sb_result= #endif GC_get_stack_base(&sb); GC_ASSERT(sb_result==GC_SUCCESS); GC_register_my_thread_inner(&sb,thread_id); #endif } break; case DLL_THREAD_DETACH: GC_ASSERT(parallel_initialized); if (GC_win32_dll_threads){ GC_delete_thread(GetCurrentThreadId()); } break; case DLL_PROCESS_DETACH: if (GC_win32_dll_threads){ int i; int my_max=(int)GC_get_max_thread_index(); for (i=0;i<=my_max;++i){ if (AO_load(&(dll_thread_table[i].tm.in_use))) GC_delete_gc_thread_no_free(&dll_thread_table[i]); } GC_deinit(); } break; } return TRUE; } #endif GC_INNER void GC_init_parallel(void) { #if defined(THREAD_LOCAL_ALLOC) GC_thread me; DCL_LOCK_STATE; #endif if (parallel_initialized)return; parallel_initialized=TRUE; if (!GC_is_initialized)GC_init(); #if defined(CPPCHECK)&&!defined(GC_NO_THREADS_DISCOVERY) GC_noop1((word)&GC_DllMain); #endif if (GC_win32_dll_threads){ set_need_to_lock(); } #if defined(THREAD_LOCAL_ALLOC) LOCK(); me=GC_lookup_thread_inner(GetCurrentThreadId()); CHECK_LOOKUP_MY_THREAD(me); GC_init_thread_local(&me->tlfs); UNLOCK(); #endif } #if defined(USE_PTHREAD_LOCKS) GC_INNER void GC_lock(void) { pthread_mutex_lock(&GC_allocate_ml); } #endif #if defined(THREAD_LOCAL_ALLOC) GC_INNER void GC_mark_thread_local_free_lists(void) { int i; GC_thread p; for (i=0;i < THREAD_TABLE_SZ;++i){ for (p=GC_threads[i];0!=p;p=p->tm.next){ if (!KNOWN_FINISHED(p)){ #ifdef DEBUG_THREADS GC_log_printf("Marking thread locals for 0x%x\n",(int)p->id); #endif GC_mark_thread_local_fls_for(&(p->tlfs)); } } } } #if defined(GC_ASSERTIONS) void GC_check_tls(void) { int i; GC_thread p; for (i=0;i < THREAD_TABLE_SZ;++i){ for (p=GC_threads[i];0!=p;p=p->tm.next){ if (!KNOWN_FINISHED(p)) GC_check_tls_for(&(p->tlfs)); } } #if defined(USE_CUSTOM_SPECIFIC) if (GC_thread_key!=0) GC_check_tsd_marks(GC_thread_key); #endif } #endif #endif #ifndef GC_NO_THREAD_REDIRECTS #define CreateThread GC_CreateThread #define ExitThread GC_ExitThread #undef _beginthreadex #define _beginthreadex GC_beginthreadex #undef _endthreadex #define _endthreadex GC_endthreadex #endif #endif #ifndef GC_PTHREAD_START_STANDALONE #if defined(__GNUC__)&&defined(__linux__) #undef __EXCEPTIONS #endif #if defined(GC_PTHREADS)&&!defined(GC_WIN32_THREADS) #include #include GC_INNER_PTHRSTART void*GC_CALLBACK GC_inner_start_routine( struct GC_stack_base*sb,void*arg) { void*(*start)(void*); void*start_arg; void*result; volatile GC_thread me= GC_start_rtn_prepare_thread(&start,&start_arg,sb,arg); #ifndef NACL pthread_cleanup_push(GC_thread_exit_proc,me); #endif result=(*start)(start_arg); #if defined(DEBUG_THREADS)&&!defined(GC_PTHREAD_START_STANDALONE) GC_log_printf("Finishing thread %p\n",(void*)pthread_self()); #endif me->status=result; GC_end_stubborn_change(me); #ifndef NACL pthread_cleanup_pop(1); #endif return result; } #endif #endif #ifndef GC_NO_THREAD_REDIRECTS #define GC_PTHREAD_REDIRECTS_ONLY #ifndef GC_PTHREAD_REDIRECTS_H #define GC_PTHREAD_REDIRECTS_H #if defined(GC_H)&&defined(GC_PTHREADS) #ifndef GC_PTHREAD_REDIRECTS_ONLY #include #ifndef GC_NO_DLOPEN #include #endif #ifndef GC_NO_PTHREAD_SIGMASK #include #endif #ifdef __cplusplus extern "C" { #endif #ifndef GC_SUSPEND_THREAD_ID #define GC_SUSPEND_THREAD_ID pthread_t #endif #ifndef GC_NO_DLOPEN GC_API void*GC_dlopen(const char*,int); #endif #ifndef GC_NO_PTHREAD_SIGMASK #if defined(GC_PTHREAD_SIGMASK_NEEDED)||defined(_BSD_SOURCE)||defined(_GNU_SOURCE)||(_POSIX_C_SOURCE>=199506L)||(_XOPEN_SOURCE>=500) GC_API int GC_pthread_sigmask(int,const sigset_t*, sigset_t*); #endif #endif #ifndef GC_PTHREAD_CREATE_CONST #define GC_PTHREAD_CREATE_CONST const #endif GC_API int GC_pthread_create(pthread_t*, GC_PTHREAD_CREATE_CONST pthread_attr_t*, void*(*)(void*),void*); GC_API int GC_pthread_join(pthread_t,void**); GC_API int GC_pthread_detach(pthread_t); #ifndef GC_NO_PTHREAD_CANCEL GC_API int GC_pthread_cancel(pthread_t); #endif #if defined(GC_HAVE_PTHREAD_EXIT)&&!defined(GC_PTHREAD_EXIT_DECLARED) #define GC_PTHREAD_EXIT_DECLARED GC_API void GC_pthread_exit(void*)GC_PTHREAD_EXIT_ATTRIBUTE; #endif #ifdef __cplusplus } #endif #endif #if!defined(GC_NO_THREAD_REDIRECTS)&&!defined(GC_USE_LD_WRAP) #undef pthread_create #undef pthread_join #undef pthread_detach #define pthread_create GC_pthread_create #define pthread_join GC_pthread_join #define pthread_detach GC_pthread_detach #ifndef GC_NO_PTHREAD_SIGMASK #undef pthread_sigmask #define pthread_sigmask GC_pthread_sigmask #endif #ifndef GC_NO_DLOPEN #undef dlopen #define dlopen GC_dlopen #endif #ifndef GC_NO_PTHREAD_CANCEL #undef pthread_cancel #define pthread_cancel GC_pthread_cancel #endif #ifdef GC_HAVE_PTHREAD_EXIT #undef pthread_exit #define pthread_exit GC_pthread_exit #endif #endif #endif #endif #endif