Fri Jan 19 10:33:42 PST 2018 Thread 6 (Thread 0x7ff9245fd700 (LWP 9097)): #0 0x00007ff928ef6b2d in accept () from /lib64/libpthread.so.0 #1 0x00007ff9246091c3 in els_accept_connection () from /sasconfig/TKGrid/lib/tkels.so #2 0x00007ff924606306 in bkelsAcceptThread () from /sasconfig/TKGrid/lib/tkels.so #3 0x00007ff927b9a494 in sktMain () from /sasconfig/TKGrid/lib/tkmk.so #4 0x00007ff927b9c231 in bktMain () from /sasconfig/TKGrid/lib/tkmk.so #5 0x00007ff928eefaa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007ff92857dbcd in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7ff9239c2700 (LWP 9098)): #0 0x00007ff928ef3a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007ff927b9c344 in bktWait () from /sasconfig/TKGrid/lib/tkmk.so #2 0x00007ff927b9ac7d in sktWaitCommon () from /sasconfig/TKGrid/lib/tkmk.so #3 0x00007ff927b9aae1 in sktWait () from /sasconfig/TKGrid/lib/tkmk.so #4 0x00007ff927bcda54 in skatAtomicMonitor () from /sasconfig/TKGrid/lib/tkmk.so #5 0x00007ff927b9a494 in sktMain () from /sasconfig/TKGrid/lib/tkmk.so #6 0x00007ff927b9c231 in bktMain () from /sasconfig/TKGrid/lib/tkmk.so #7 0x00007ff928eefaa1 in start_thread () from /lib64/libpthread.so.0 #8 0x00007ff92857dbcd in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7ff91be93700 (LWP 9099)): #0 0x00007ff928ef3a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007ff927b9c344 in bktWait () from /sasconfig/TKGrid/lib/tkmk.so #2 0x00007ff927b9ac7d in sktWaitCommon () from /sasconfig/TKGrid/lib/tkmk.so #3 0x00007ff927b9aae1 in sktWait () from /sasconfig/TKGrid/lib/tkmk.so #4 0x00007ff92954523a in watchAbort () #5 0x00007ff927b9a494 in sktMain () from /sasconfig/TKGrid/lib/tkmk.so #6 0x00007ff927b9c231 in bktMain () from /sasconfig/TKGrid/lib/tkmk.so #7 0x00007ff928eefaa1 in start_thread () from /lib64/libpthread.so.0 #8 0x00007ff92857dbcd in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x7ff91a3ef700 (LWP 9100)): #0 0x00007ff928574383 in poll () from /lib64/libc.so.6 #1 0x00007ff91a646f7e in bkSelectWait () from /sasconfig/TKGrid/lib/tkitcp.so #2 0x00007ff91a641673 in sktSelectSelectNoLock () from /sasconfig/TKGrid/lib/tkitcp.so #3 0x00007ff91a63a003 in TKTcpEventPostThread () from /sasconfig/TKGrid/lib/tkitcp.so #4 0x00007ff927b9a494 in sktMain () from /sasconfig/TKGrid/lib/tkmk.so #5 0x00007ff927b9c231 in bktMain () from /sasconfig/TKGrid/lib/tkmk.so #6 0x00007ff928eefaa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007ff92857dbcd in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x7ff916be9700 (LWP 9101)): #0 0x00007ff928541c4d in nanosleep () from /lib64/libc.so.6 #1 0x00007ff928541ac0 in sleep () from /lib64/libc.so.6 #2 0x00007ff929543878 in debugMyself2 () #3 0x00007ff91c78523d in MPIR_Err_return_comm () from /sasconfig/TKGrid/mpich2-install/lib/libmpich.so #4 0x00007ff91c7cd1d0 in PMPI_Recv () from /sasconfig/TKGrid/mpich2-install/lib/libmpich.so #5 0x00007ff91adf245e in MPIServerThread () from /sasconfig/TKGrid/lib/lasr.so #6 0x00007ff927b9a494 in sktMain () from /sasconfig/TKGrid/lib/tkmk.so #7 0x00007ff927b9c231 in bktMain () from /sasconfig/TKGrid/lib/tkmk.so #8 0x00007ff928eefaa1 in start_thread () from /lib64/libpthread.so.0 #9 0x00007ff92857dbcd in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7ff929a01700 (LWP 9095)): #0 0x00007ff928ef3a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007ff927b9c344 in bktWait () from /sasconfig/TKGrid/lib/tkmk.so #2 0x00007ff927b9ac7d in sktWaitCommon () from /sasconfig/TKGrid/lib/tkmk.so #3 0x00007ff927b9aae1 in sktWait () from /sasconfig/TKGrid/lib/tkmk.so #4 0x00007ff91adf3528 in SocketServer () from /sasconfig/TKGrid/lib/lasr.so #5 0x00007ff91acea75a in gridmain () from /sasconfig/TKGrid/lib/lasr.so #6 0x00007ff929543b23 in rungridmain () #7 0x00007ff927bd286b in skCallMorphed () from /sasconfig/TKGrid/lib/tkmk.so #8 0x00007ff92954038b in main () Fri Jan 26 09:31:14 PST 2018 Thread 6 (Thread 0x7faaf5c2f700 (LWP 31904)): #0 0x00007faafa528b2d in accept () from /lib64/libpthread.so.0 #1 0x00007faaf5c3b1c3 in els_accept_connection () from /sasconfig/TKGrid/lib/tkels.so #2 0x00007faaf5c38306 in bkelsAcceptThread () from /sasconfig/TKGrid/lib/tkels.so #3 0x00007faaf91cc494 in sktMain () from /sasconfig/TKGrid/lib/tkmk.so #4 0x00007faaf91ce231 in bktMain () from /sasconfig/TKGrid/lib/tkmk.so #5 0x00007faafa521aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007faaf9bafbcd in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7faaf4ff4700 (LWP 31905)): #0 0x00007faafa525a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007faaf91ce344 in bktWait () from /sasconfig/TKGrid/lib/tkmk.so #2 0x00007faaf91ccc7d in sktWaitCommon () from /sasconfig/TKGrid/lib/tkmk.so #3 0x00007faaf91ccae1 in sktWait () from /sasconfig/TKGrid/lib/tkmk.so #4 0x00007faaf91ffa54 in skatAtomicMonitor () from /sasconfig/TKGrid/lib/tkmk.so #5 0x00007faaf91cc494 in sktMain () from /sasconfig/TKGrid/lib/tkmk.so #6 0x00007faaf91ce231 in bktMain () from /sasconfig/TKGrid/lib/tkmk.so #7 0x00007faafa521aa1 in start_thread () from /lib64/libpthread.so.0 #8 0x00007faaf9bafbcd in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7faaed4c5700 (LWP 31906)): #0 0x00007faafa528334 in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007faafa5235d8 in _L_lock_854 () from /lib64/libpthread.so.0 #2 0x00007faafa5234a7 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007faaeddd748d in MPIU_Thread_CS_enter_lockname_recursive_impl_ () from /sasconfig/TKGrid/mpich2-install/lib/libmpich.so #4 0x00007faaeddd791a in PMPI_Iprobe () from /sasconfig/TKGrid/mpich2-install/lib/libmpich.so #5 0x00007faafab77213 in watchAbort () #6 0x00007faaf91cc494 in sktMain () from /sasconfig/TKGrid/lib/tkmk.so #7 0x00007faaf91ce231 in bktMain () from /sasconfig/TKGrid/lib/tkmk.so #8 0x00007faafa521aa1 in start_thread () from /lib64/libpthread.so.0 #9 0x00007faaf9bafbcd in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x7faaeba21700 (LWP 31907)): #0 0x00007faaf9ba6383 in poll () from /lib64/libc.so.6 #1 0x00007faaebc78f7e in bkSelectWait () from /sasconfig/TKGrid/lib/tkitcp.so #2 0x00007faaebc73673 in sktSelectSelectNoLock () from /sasconfig/TKGrid/lib/tkitcp.so #3 0x00007faaebc6c003 in TKTcpEventPostThread () from /sasconfig/TKGrid/lib/tkitcp.so #4 0x00007faaf91cc494 in sktMain () from /sasconfig/TKGrid/lib/tkmk.so #5 0x00007faaf91ce231 in bktMain () from /sasconfig/TKGrid/lib/tkmk.so #6 0x00007faafa521aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007faaf9bafbcd in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x7faae821b700 (LWP 31908)): #0 0x00007faaf9b73c4d in nanosleep () from /lib64/libc.so.6 #1 0x00007faaf9b73ac0 in sleep () from /lib64/libc.so.6 #2 0x00007faafab75878 in debugMyself2 () #3 0x00007faaeddb723d in MPIR_Err_return_comm () from /sasconfig/TKGrid/mpich2-install/lib/libmpich.so #4 0x00007faaeddff1d0 in PMPI_Recv () from /sasconfig/TKGrid/mpich2-install/lib/libmpich.so #5 0x00007faaec42445e in MPIServerThread () from /sasconfig/TKGrid/lib/lasr.so #6 0x00007faaf91cc494 in sktMain () from /sasconfig/TKGrid/lib/tkmk.so #7 0x00007faaf91ce231 in bktMain () from /sasconfig/TKGrid/lib/tkmk.so #8 0x00007faafa521aa1 in start_thread () from /lib64/libpthread.so.0 #9 0x00007faaf9bafbcd in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7faafb033700 (LWP 31902)): #0 0x00007faafa525a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007faaf91ce344 in bktWait () from /sasconfig/TKGrid/lib/tkmk.so #2 0x00007faaf91ccc7d in sktWaitCommon () from /sasconfig/TKGrid/lib/tkmk.so #3 0x00007faaf91ccae1 in sktWait () from /sasconfig/TKGrid/lib/tkmk.so #4 0x00007faaec425528 in SocketServer () from /sasconfig/TKGrid/lib/lasr.so #5 0x00007faaec31c75a in gridmain () from /sasconfig/TKGrid/lib/lasr.so #6 0x00007faafab75b23 in rungridmain () #7 0x00007faaf920486b in skCallMorphed () from /sasconfig/TKGrid/lib/tkmk.so #8 0x00007faafab7238b in main ()