Sleeping for a random period of time up to 10 minutes

Sleeping for 135.35404895418168 seconds

Mon May 12 16:32:17 UTC 2025

# how long have we been up

uptime
16:32:17 up 41 min, 0 users, load average: 0.04, 0.03, 0.01

# whats running, with ASCII-art process hierarchy (forest)

if [ "$dumpType" == "full" ]
then
ps auxwwwf
else
ps auxf --width=200
fi
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 2 0.0 0.0 0 0 ? S 15:50 0:00 [kthreadd]
root 4 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/0:0H]
root 5 0.0 0.0 0 0 ? I 15:50 0:00 \_
[kworker/u16:0]
root 6 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [mm_percpu_wq]
root 7 0.0 0.0 0 0 ? S 15:50 0:00 \_ [ksoftirqd/0]
root 8 0.0 0.0 0 0 ? I 15:50 0:00 \_ [rcu_sched]
root 9 0.0 0.0 0 0 ? I 15:50 0:00 \_ [rcu_bh]
root 10 0.0 0.0 0 0 ? S 15:50 0:00 \_ [migration/0]
root 11 0.0 0.0 0 0 ? S 15:50 0:00 \_ [cpuhp/0]
root 12 0.0 0.0 0 0 ? S 15:50 0:00 \_ [cpuhp/1]
root 13 0.0 0.0 0 0 ? S 15:50 0:00 \_ [migration/1]
root 14 0.0 0.0 0 0 ? S 15:50 0:00 \_ [ksoftirqd/1]
root 16 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/1:0H]
root 17 0.0 0.0 0 0 ? S 15:50 0:00 \_ [cpuhp/2]
root 18 0.0 0.0 0 0 ? S 15:50 0:00 \_ [migration/2]
root 19 0.0 0.0 0 0 ? S 15:50 0:00 \_ [ksoftirqd/2]
root 21 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/2:0H]
root 22 0.0 0.0 0 0 ? S 15:50 0:00 \_ [cpuhp/3]
root 23 0.0 0.0 0 0 ? S 15:50 0:00 \_ [migration/3]
root 24 0.0 0.0 0 0 ? S 15:50 0:00 \_ [ksoftirqd/3]
root 26 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/3:0H]
root 27 0.0 0.0 0 0 ? S 15:50 0:00 \_ [cpuhp/4]
root 28 0.0 0.0 0 0 ? S 15:50 0:00 \_ [migration/4]
root 29 0.0 0.0 0 0 ? S 15:50 0:00 \_ [ksoftirqd/4]
root 31 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/4:0H]
root 32 0.0 0.0 0 0 ? S 15:50 0:00 \_ [cpuhp/5]
root 33 0.0 0.0 0 0 ? S 15:50 0:00 \_ [migration/5]
root 34 0.0 0.0 0 0 ? S 15:50 0:00 \_ [ksoftirqd/5]
root 36 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/5:0H]
root 37 0.0 0.0 0 0 ? S 15:50 0:00 \_ [cpuhp/6]
root 38 0.0 0.0 0 0 ? S 15:50 0:00 \_ [migration/6]
root 39 0.0 0.0 0 0 ? S 15:50 0:00 \_ [ksoftirqd/6]
root 41 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/6:0H]
root 42 0.0 0.0 0 0 ? S 15:50 0:00 \_ [cpuhp/7]
root 43 0.0 0.0 0 0 ? S 15:50 0:00 \_ [migration/7]
root 44 0.0 0.0 0 0 ? S 15:50 0:00 \_ [ksoftirqd/7]
root 46 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/7:0H]
root 47 0.0 0.0 0 0 ? S 15:50 0:00 \_ [kdevtmpfs]
root 48 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [netns]
root 51 0.0 0.0 0 0 ? S 15:50 0:00 \_ [khungtaskd]
root 52 0.0 0.0 0 0 ? S 15:50 0:00 \_ [oom_reaper]
root 53 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [writeback]
root 54 0.0 0.0 0 0 ? S 15:50 0:00 \_ [kcompactd0]
root 55 0.0 0.0 0 0 ? SN 15:50 0:00 \_ [ksmd]
root 56 0.0 0.0 0 0 ? SN 15:50 0:00 \_ [khugepaged]
root 57 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [crypto]
root 58 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kintegrityd]
root 59 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kblockd]
root 60 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [ata_sff]
root 61 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [md]
root 62 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [edac-poller]
root 63 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [devfreq_wq]
root 64 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [watchdogd]
root 78 0.0 0.0 0 0 ? I 15:50 0:00 \_
[kworker/u16:1]
root 131 0.0 0.0 0 0 ? S 15:50 0:00 \_ [kauditd]
root 132 0.0 0.0 0 0 ? S 15:50 0:00 \_ [kswapd0]
root 133 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [xfsalloc]
root 134 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs_mru_cache]
root 180 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kthrotld]
root 181 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/10-
ACPI:Ged]
root 182 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/11-
ACPI:Ged]
root 183 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/12-
ACPI:Ged]
root 184 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/13-
ACPI:Ged]
root 185 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/14-
ACPI:Ged]
root 186 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/15-
ACPI:Ged]
root 187 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/16-
ACPI:Ged]
root 188 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/17-
ACPI:Ged]
root 189 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/18-
ACPI:Ged]
root 190 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/19-
ACPI:Ged]
root 191 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/20-
ACPI:Ged]
root 192 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/21-
ACPI:Ged]
root 193 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/22-
ACPI:Ged]
root 194 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/23-
ACPI:Ged]
root 195 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/24-
ACPI:Ged]
root 196 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/25-
ACPI:Ged]
root 197 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/26-
ACPI:Ged]
root 198 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/27-
ACPI:Ged]
root 199 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/28-
ACPI:Ged]
root 200 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/29-
ACPI:Ged]
root 201 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/30-
ACPI:Ged]
root 202 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/31-
ACPI:Ged]
root 203 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/32-
ACPI:Ged]
root 204 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/33-
ACPI:Ged]
root 205 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/34-
ACPI:Ged]
root 206 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/35-
ACPI:Ged]
root 207 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/36-
ACPI:Ged]
root 208 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/37-
ACPI:Ged]
root 209 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/38-
ACPI:Ged]
root 210 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/39-
ACPI:Ged]
root 211 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/40-
ACPI:Ged]
root 212 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/41-
ACPI:Ged]
root 213 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[acpi_thermal_pm]
root 214 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kmpath_rdacd]
root 215 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kaluad]
root 216 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [nvme-wq]
root 218 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[ipv6_addrconf]
root 219 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kstrp]
root 229 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/43-
ACPI:Eve]
root 230 0.0 0.0 0 0 ? S 15:50 0:00 \_ [irq/44-
ACPI:Eve]
root 323 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-buf/nvme0n1]
root 324 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-data/nvme0n]
root 325 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-conv/nvme0n]
root 326 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-cil/nvme0n1]
root 327 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-reclaim/nvm]
root 328 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-log/nvme0n1]
root 329 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [xfs-
eofblocks/n]
root 330 0.0 0.0 0 0 ? S 15:50 0:00 \_
[xfsaild/nvme0n1]
root 331 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/0:1H]
root 424 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/4:1H]
root 435 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [ena]
root 555 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [rpciod]
root 556 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [xprtiod]
root 1170 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/5:1H]
root 1171 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/6:1H]
root 1172 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/7:1H]
root 1223 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/2:1H]
root 1304 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-buf/nvme1n1]
root 1305 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-data/nvme1n]
root 1306 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-conv/nvme1n]
root 1307 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-cil/nvme1n1]
root 1308 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-reclaim/nvm]
root 1309 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-log/nvme1n1]
root 1310 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [xfs-
eofblocks/n]
root 1319 0.0 0.0 0 0 ? S 15:50 0:00 \_
[xfsaild/nvme1n1]
root 1349 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-buf/nvme1n1]
root 1350 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-data/nvme1n]
root 1351 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-conv/nvme1n]
root 1352 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-cil/nvme1n1]
root 1353 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-reclaim/nvm]
root 1354 0.0 0.0 0 0 ? I< 15:50 0:00 \_
[xfs-log/nvme1n1]
root 1355 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [xfs-
eofblocks/n]
root 1361 0.0 0.0 0 0 ? S 15:50 0:00 \_
[xfsaild/nvme1n1]
root 2556 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/1:1H]
root 2559 0.0 0.0 0 0 ? I< 15:50 0:00 \_ [kworker/3:1H]
root 19940 0.0 0.0 0 0 ? I 16:06 0:00 \_ [kworker/4:0]
root 24161 0.0 0.0 0 0 ? I 16:11 0:00 \_ [kworker/1:1]
root 25320 0.0 0.0 0 0 ? I 16:12 0:00 \_ [kworker/3:2]
root 26707 0.0 0.0 0 0 ? I 16:15 0:00 \_ [kworker/2:0]
root 30622 0.0 0.0 0 0 ? I 16:20 0:00 \_ [kworker/5:2]
root 31741 0.0 0.0 0 0 ? I 16:21 0:00 \_ [kworker/7:2]
root 31768 0.0 0.0 0 0 ? I 16:21 0:00 \_ [kworker/6:0]
root 32279 0.0 0.0 0 0 ? I 16:22 0:00 \_ [kworker/0:1]
root 753 0.0 0.0 0 0 ? I 16:23 0:00 \_ [kworker/7:3]
root 777 0.0 0.0 0 0 ? I 16:23 0:00 \_ [kworker/4:1]
root 1117 0.0 0.0 0 0 ? I 16:23 0:00 \_ [kworker/3:1]
root 1517 0.0 0.0 0 0 ? I 16:23 0:00 \_ [kworker/0:3]
root 1906 0.0 0.0 0 0 ? I 16:23 0:00 \_ [kworker/6:2]
root 2451 0.0 0.0 0 0 ? I 16:23 0:00 \_ [kworker/2:2]
root 4467 0.0 0.0 0 0 ? I 16:26 0:00 \_ [kworker/5:3]
root 4548 0.0 0.0 0 0 ? I 16:26 0:00 \_ [kworker/1:0]
root 6274 0.0 0.0 0 0 ? I 16:28 0:00 \_ [kworker/3:0]
root 6353 0.0 0.0 0 0 ? I 16:29 0:00 \_ [kworker/2:1]
root 6957 0.0 0.0 0 0 ? I 16:30 0:00 \_ [kworker/7:0]
root 8053 0.0 0.0 0 0 ? I 16:31 0:00 \_ [kworker/6:1]
root 8056 0.0 0.0 0 0 ? I 16:31 0:00 \_ [kworker/7:1]
root 8111 0.0 0.0 0 0 ? I 16:31 0:00 \_ [kworker/2:3]
root 8112 0.0 0.0 0 0 ? I 16:31 0:00 \_ [kworker/4:2]
root 8219 0.0 0.0 0 0 ? I 16:31 0:00 \_ [kworker/0:0]
root 1 0.0 0.0 8508 5428 ? Ss 15:50 0:01
/usr/lib/systemd/systemd --switched-root --system --deserialize 21
root 394 0.0 0.0 14432 9072 ? Ss 15:50 0:00
/usr/lib/systemd/systemd-journald
root 421 0.0 0.0 82640 1608 ? Ss 15:50 0:00 /usr/sbin/lvmetad
-f
root 434 0.0 0.0 14832 3456 ? Ss 15:50 0:00
/usr/lib/systemd/systemd-udevd
root 558 0.0 0.0 17840 2088 ? S<sl 15:50 0:00 /sbin/auditd
dbus 582 0.0 0.0 8592 3712 ? Ss 15:50 0:00 /usr/bin/dbus-
daemon --system --address=systemd: --nofork --nopidfile --systemd-activation
root 588 0.0 0.0 4840 2780 ? Ss 15:50 0:00
/usr/lib/systemd/systemd-logind
root 589 0.0 0.0 81584 2560 ? Ssl 15:50 0:00
/usr/sbin/irqbalance --foreground
libstor+ 590 0.0 0.0 2568 1536 ? Ss 15:50 0:00 /usr/bin/lsmd -d
root 592 0.0 0.0 2040 104 ? Ss 15:50 0:00 /usr/sbin/acpid
rngd 600 0.7 0.0 322336 4064 ? Ssl 15:50 0:19 /sbin/rngd -f --
fill-watermark=0
root 605 0.0 0.0 58516 2544 ? Ssl 15:50 0:00 /usr/sbin/gssproxy
-D
root 1013 0.0 0.0 21104 4560 ? Ss 15:50 0:00
/usr/libexec/postfix/master -w
postfix 1014 0.0 0.0 21196 6468 ? S 15:50 0:00 \_ pickup -l -t
unix -u
postfix 1015 0.0 0.0 21276 6412 ? S 15:50 0:00 \_ qmgr -l -t
unix -u
root 2952 0.0 0.0 19708 3628 ? Ss 15:50 0:00 /sbin/dhclient -q
-lf /var/lib/dhclient/dhclient--eth0.lease -pf /var/run/dhclient-eth0.pid -H ip-10-
20-77-92 eth0
root 3029 0.0 0.0 19708 4228 ? Ss 15:50 0:00 /sbin/dhclient -6
-nw -lf /var/lib/dhclient/dhclient6--eth0.lease -pf /var/run/dhclient6-eth0.pid
eth0 -H ip-10-20-77-92
chrony 3230 0.0 0.0 80528 2740 ? S 15:50 0:00 /usr/sbin/chronyd
-F 2
root 3314 0.0 0.0 175412 6664 ? Ssl 15:50 0:00 /usr/sbin/rsyslogd
-n
root 3315 0.0 0.1 1242896 17428 ? Ssl 15:50 0:00 /usr/bin/amazon-
ssm-agent
root 3337 0.0 0.0 4480 2472 ? Ss 15:50 0:00 /usr/sbin/crond -n
root 6947 0.0 0.0 9304 3288 ? S 16:30 0:00 \_
/usr/sbin/CROND -n
root 6949 0.0 0.0 3376 596 ? Ss 16:30 0:00 \_ /bin/sh -c
/usr/bin/dump-instance-state full 10 > /emr/instance-state/instance-state.log-`date
+%Y-%m-%d-%H-%M` 2>&1
root 6950 0.0 0.0 3376 2148 ? S 16:30 0:00 \_
/bin/bash /usr/bin/dump-instance-state full 10
root 8838 0.0 0.0 8360 3376 ? R 16:32 0:00 \_ ps
auxwwwf
root 3343 0.0 0.0 4016 2008 ? Ss 15:50 0:00 /usr/sbin/atd -f
root 3353 0.0 0.0 113216 1704 tty1 Ss+ 15:50 0:00 /sbin/agetty --
noclear tty1 linux
root 3354 0.0 0.0 2464 1692 ttyS0 Ss+ 15:50 0:00 /sbin/agetty --
keep-baud 115200,38400,9600 ttyS0 vt220
root 3391 0.0 0.0 18264 6836 ? Ss 15:50 0:00 /usr/sbin/sshd -D
root 3633 0.0 0.0 3664 2328 ? Ss 15:51 0:00 /bin/bash
/usr/bin/ic-heartbeat-monitor
root 8776 0.0 0.0 2012 348 ? S 16:32 0:00 \_ sleep 10
root 3832 0.5 1.6 4843680 264384 ? Sl 15:51 0:14
/etc/alternatives/jre/bin/java -Xmx1024m -XX:+ExitOnOutOfMemoryError -
XX:MinHeapFreeRatio=10 -server -cp
/usr/share/aws/emr/instance-controller/lib/*:/home/hadoop/conf -
Dlog4j.defaultInitOverride aws157.logpusher.Main
/etc/logpusher/logpusher.properties
root 3879 0.2 1.3 4736404 211512 ? Sl 15:51 0:05 /usr/bin/java -
Xmx1500m -Xms300m -XX:+ExitOnOutOfMemoryError -XX:MinHeapFreeRatio=10 -cp
/usr/share/aws/emr/emr-log-analytics-metrics/lib/*:/home/hadoop/conf -
Dlog4j.defaultInitOverride aws157.apppusher.Main
hadoop 3946 1.0 1.9 6400620 314656 ? Sl 15:51 0:26
/etc/alternatives/jre/bin/java -Djava.security.properties=/emr/instance-
controller/lib/security.properties -Xmx1024m -Djute.maxbuffer=3145728 -XX:
+ExitOnOutOfMemoryError -XX:MinHeapFreeRatio=10 -server -cp
/usr/share/aws/emr/instance-controller/lib/*:/home/hadoop/conf -
Dlog4j.defaultInitOverride aws157.instancecontroller.Main
root 7442 0.2 0.2 2332960 43508 ? Ssl 15:52 0:05
/usr/bin/containerd
root 7488 0.0 0.4 2286320 73392 ? Ssl 15:52 0:00 /usr/bin/dockerd -
H fd:// --containerd=/run/containerd/containerd.sock --default-ulimit
nofile=32768:65536
hdfs 8234 0.3 1.6 2555844 264444 ? Sl 15:52 0:08
/etc/alternatives/jre/bin/java -Dproc_datanode -Djava.net.preferIPv4Stack=true -
server -XX:+ExitOnOutOfMemoryError -Dhadoop.security.logger=ERROR,DRFAS -
Dyarn.log.dir=/var/log/hadoop-hdfs -Dyarn.log.file=hadoop-hdfs-datanode-ip-10-20-
77-92.log -Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console -
Djava.library.path=:/usr/lib/hadoop-lzo/lib/native:/usr/lib/hadoop/lib/native -
Xmx762m -Dhadoop.log.dir=/var/log/hadoop-hdfs -Dhadoop.log.file=hadoop-hdfs-
datanode-ip-10-20-77-92.log -Dhadoop.home.dir=/usr/lib/hadoop -Dhadoop.id.str=hdfs
-Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-policy.xml -
Dsun.net.inetaddr.ttl=3600 org.apache.hadoop.hdfs.server.datanode.DataNode
yarn 8572 0.8 2.8 4139452 462680 ? Sl 15:52 0:20
/etc/alternatives/jre/bin/java -Dproc_nodemanager -Djava.net.preferIPv4Stack=true -
server -XX:+ExitOnOutOfMemoryError -XX:+ExitOnOutOfMemoryError -
Dsun.net.inetaddr.ttl=30 -Dyarn.log.dir=/var/log/hadoop-yarn -
Dyarn.log.file=hadoop-yarn-nodemanager-ip-10-20-77-92.log
-Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console -
Djava.library.path=:/usr/lib/hadoop-lzo/lib/native:/usr/lib/hadoop/lib/native -
Xmx2048m -Dhadoop.log.dir=/var/log/hadoop-yarn -Dhadoop.log.file=hadoop-yarn-
nodemanager-ip-10-20-77-92.log -Dhadoop.home.dir=/usr/lib/hadoop -
Dhadoop.id.str=yarn -Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-
policy.xml -Dhadoop.security.logger=INFO,NullAppender
org.apache.hadoop.yarn.server.nodemanager.NodeManager
root 16331 0.0 0.0 3260 144 ? Ss 16:01 0:00 /usr/sbin/anacron
-s
root 351 0.0 0.0 3928 2700 ? Ss 16:22 0:00 /bin/bash
/usr/bin/log4j-cve-2021-44228-hotpatch -w 1800 -m 10
root 8811 0.0 0.0 2012 404 ? S 16:32 0:00 \_ sleep 1

# Top CPU users

ps auxwww --sort -%cpu | head -20
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
hadoop 3946 1.0 1.9 6400620 314656 ? Sl 15:51 0:26
/etc/alternatives/jre/bin/java -Djava.security.properties=/emr/instance-
controller/lib/security.properties -Xmx1024m -Djute.maxbuffer=3145728 -XX:
+ExitOnOutOfMemoryError -XX:MinHeapFreeRatio=10 -server -cp
/usr/share/aws/emr/instance-controller/lib/*:/home/hadoop/conf -
Dlog4j.defaultInitOverride aws157.instancecontroller.Main
yarn 8572 0.8 2.8 4139452 462680 ? Sl 15:52 0:20
/etc/alternatives/jre/bin/java -Dproc_nodemanager -Djava.net.preferIPv4Stack=true -
server -XX:+ExitOnOutOfMemoryError -XX:+ExitOnOutOfMemoryError -
Dsun.net.inetaddr.ttl=30 -Dyarn.log.dir=/var/log/hadoop-yarn -
Dyarn.log.file=hadoop-yarn-nodemanager-ip-10-20-77-92.log
-Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console -
Djava.library.path=:/usr/lib/hadoop-lzo/lib/native:/usr/lib/hadoop/lib/native -
Xmx2048m -Dhadoop.log.dir=/var/log/hadoop-yarn -Dhadoop.log.file=hadoop-yarn-
nodemanager-ip-10-20-77-92.log -Dhadoop.home.dir=/usr/lib/hadoop -
Dhadoop.id.str=yarn -Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-
policy.xml -Dhadoop.security.logger=INFO,NullAppender
org.apache.hadoop.yarn.server.nodemanager.NodeManager
rngd 600 0.7 0.0 322336 4064 ? Ssl 15:50 0:19 /sbin/rngd -f --
fill-watermark=0
root 3832 0.5 1.6 4843680 264384 ? Sl 15:51 0:14
/etc/alternatives/jre/bin/java -Xmx1024m -XX:+ExitOnOutOfMemoryError -
XX:MinHeapFreeRatio=10 -server -cp
/usr/share/aws/emr/instance-controller/lib/*:/home/hadoop/conf -
Dlog4j.defaultInitOverride aws157.logpusher.Main
/etc/logpusher/logpusher.properties
hdfs 8234 0.3 1.6 2555844 264444 ? Sl 15:52 0:08
/etc/alternatives/jre/bin/java -Dproc_datanode -Djava.net.preferIPv4Stack=true -
server -XX:+ExitOnOutOfMemoryError -Dhadoop.security.logger=ERROR,DRFAS -
Dyarn.log.dir=/var/log/hadoop-hdfs -Dyarn.log.file=hadoop-hdfs-datanode-ip-10-20-
77-92.log -Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console -
Djava.library.path=:/usr/lib/hadoop-lzo/lib/native:/usr/lib/hadoop/lib/native -
Xmx762m -Dhadoop.log.dir=/var/log/hadoop-hdfs -Dhadoop.log.file=hadoop-hdfs-
datanode-ip-10-20-77-92.log -Dhadoop.home.dir=/usr/lib/hadoop -Dhadoop.id.str=hdfs
-Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-policy.xml -
Dsun.net.inetaddr.ttl=3600 org.apache.hadoop.hdfs.server.datanode.DataNode
root 3879 0.2 1.3 4736404 211512 ? Sl 15:51 0:05 /usr/bin/java -
Xmx1500m -Xms300m -XX:+ExitOnOutOfMemoryError -XX:MinHeapFreeRatio=10 -cp
/usr/share/aws/emr/emr-log-analytics-metrics/lib/*:/home/hadoop/conf -
Dlog4j.defaultInitOverride aws157.apppusher.Main
root 7442 0.2 0.2 2332960 43508 ? Ssl 15:52 0:05
/usr/bin/containerd
root 1 0.0 0.0 8508 5428 ? Ss 15:50 0:01
/usr/lib/systemd/systemd --switched-root --system --deserialize 21
root 2 0.0 0.0 0 0 ? S 15:50 0:00 [kthreadd]
root 4 0.0 0.0 0 0 ? I< 15:50 0:00 [kworker/0:0H]
root 5 0.0 0.0 0 0 ? I 15:50 0:00 [kworker/u16:0]
root 6 0.0 0.0 0 0 ? I< 15:50 0:00 [mm_percpu_wq]
root 7 0.0 0.0 0 0 ? S 15:50 0:00 [ksoftirqd/0]
root 8 0.0 0.0 0 0 ? I 15:50 0:00 [rcu_sched]
root 9 0.0 0.0 0 0 ? I 15:50 0:00 [rcu_bh]
root 10 0.0 0.0 0 0 ? S 15:50 0:00 [migration/0]
root 11 0.0 0.0 0 0 ? S 15:50 0:00 [cpuhp/0]
root 12 0.0 0.0 0 0 ? S 15:50 0:00 [cpuhp/1]
root 13 0.0 0.0 0 0 ? S 15:50 0:00 [migration/1]

# Top memory users

ps auxwww --sort -rss | head -20
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
yarn 8572 0.8 2.8 4139452 462680 ? Sl 15:52 0:20
/etc/alternatives/jre/bin/java -Dproc_nodemanager -Djava.net.preferIPv4Stack=true -
server -XX:+ExitOnOutOfMemoryError -XX:+ExitOnOutOfMemoryError -
Dsun.net.inetaddr.ttl=30 -Dyarn.log.dir=/var/log/hadoop-yarn -
Dyarn.log.file=hadoop-yarn-nodemanager-ip-10-20-77-92.log
-Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console -
Djava.library.path=:/usr/lib/hadoop-lzo/lib/native:/usr/lib/hadoop/lib/native -
Xmx2048m -Dhadoop.log.dir=/var/log/hadoop-yarn -Dhadoop.log.file=hadoop-yarn-
nodemanager-ip-10-20-77-92.log -Dhadoop.home.dir=/usr/lib/hadoop -
Dhadoop.id.str=yarn -Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-
policy.xml -Dhadoop.security.logger=INFO,NullAppender
org.apache.hadoop.yarn.server.nodemanager.NodeManager
hadoop 3946 1.0 1.9 6400620 314656 ? Sl 15:51 0:26
/etc/alternatives/jre/bin/java -Djava.security.properties=/emr/instance-
controller/lib/security.properties -Xmx1024m -Djute.maxbuffer=3145728 -XX:
+ExitOnOutOfMemoryError -XX:MinHeapFreeRatio=10 -server -cp
/usr/share/aws/emr/instance-controller/lib/*:/home/hadoop/conf -
Dlog4j.defaultInitOverride aws157.instancecontroller.Main
hdfs 8234 0.3 1.6 2555844 264444 ? Sl 15:52 0:08
/etc/alternatives/jre/bin/java -Dproc_datanode -Djava.net.preferIPv4Stack=true -
server -XX:+ExitOnOutOfMemoryError -Dhadoop.security.logger=ERROR,DRFAS -
Dyarn.log.dir=/var/log/hadoop-hdfs -Dyarn.log.file=hadoop-hdfs-datanode-ip-10-20-
77-92.log -Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console -
Djava.library.path=:/usr/lib/hadoop-lzo/lib/native:/usr/lib/hadoop/lib/native -
Xmx762m -Dhadoop.log.dir=/var/log/hadoop-hdfs -Dhadoop.log.file=hadoop-hdfs-
datanode-ip-10-20-77-92.log -Dhadoop.home.dir=/usr/lib/hadoop -Dhadoop.id.str=hdfs
-Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-policy.xml -
Dsun.net.inetaddr.ttl=3600 org.apache.hadoop.hdfs.server.datanode.DataNode
root 3832 0.5 1.6 4843680 264384 ? Sl 15:51 0:14
/etc/alternatives/jre/bin/java -Xmx1024m -XX:+ExitOnOutOfMemoryError -
XX:MinHeapFreeRatio=10 -server -cp
/usr/share/aws/emr/instance-controller/lib/*:/home/hadoop/conf -
Dlog4j.defaultInitOverride aws157.logpusher.Main
/etc/logpusher/logpusher.properties
root 3879 0.2 1.3 4736404 211512 ? Sl 15:51 0:05 /usr/bin/java -
Xmx1500m -Xms300m -XX:+ExitOnOutOfMemoryError -XX:MinHeapFreeRatio=10 -cp
/usr/share/aws/emr/emr-log-analytics-metrics/lib/*:/home/hadoop/conf -
Dlog4j.defaultInitOverride aws157.apppusher.Main
root 7488 0.0 0.4 2286320 73392 ? Ssl 15:52 0:00 /usr/bin/dockerd -
H fd:// --containerd=/run/containerd/containerd.sock --default-ulimit
nofile=32768:65536
root 7442 0.2 0.2 2332960 43508 ? Ssl 15:52 0:05
/usr/bin/containerd
root 3315 0.0 0.1 1242896 17428 ? Ssl 15:50 0:00 /usr/bin/amazon-
ssm-agent
root 394 0.0 0.0 14432 9072 ? Ss 15:50 0:00
/usr/lib/systemd/systemd-journald
root 3391 0.0 0.0 18264 6836 ? Ss 15:50 0:00 /usr/sbin/sshd -D
root 3314 0.0 0.0 175412 6664 ? Ssl 15:50 0:00 /usr/sbin/rsyslogd
-n
postfix 1014 0.0 0.0 21196 6468 ? S 15:50 0:00 pickup -l -t unix
-u
postfix 1015 0.0 0.0 21276 6412 ? S 15:50 0:00 qmgr -l -t unix -u
root 1 0.0 0.0 8508 5428 ? Ss 15:50 0:01
/usr/lib/systemd/systemd --switched-root --system --deserialize 21
root 1013 0.0 0.0 21104 4560 ? Ss 15:50 0:00
/usr/libexec/postfix/master -w
root 3029 0.0 0.0 19708 4228 ? Ss 15:50 0:00 /sbin/dhclient -6
-nw -lf /var/lib/dhclient/dhclient6--eth0.lease -pf /var/run/dhclient6-eth0.pid
eth0 -H ip-10-20-77-92
rngd 600 0.7 0.0 322336 4064 ? Ssl 15:50 0:19 /sbin/rngd -f --
fill-watermark=0
dbus 582 0.0 0.0 8592 3712 ? Ss 15:50 0:00 /usr/bin/dbus-
daemon --system --address=systemd: --nofork --nopidfile --systemd-activation
root 2952 0.0 0.0 19708 3628 ? Ss 15:50 0:00 /sbin/dhclient -q
-lf /var/lib/dhclient/dhclient--eth0.lease -pf /var/run/dhclient-eth0.pid -H ip-10-
20-77-92 eth0

# hows the kernel looking

dmesg | tail -n 25
[ 2.362672] device-mapper: ioctl: 4.37.0-ioctl (2017-09-20) initialised: dm-
[email protected]
[ 2.537753] RPC: Registered named UNIX socket transport module.
[ 2.538684] RPC: Registered udp transport module.
[ 2.539433] RPC: Registered tcp transport module.
[ 2.540189] RPC: Registered tcp NFSv4.1 backchannel transport module.
[ 3.604313] ena 0000:00:05.0 eth0: Local page cache is disabled for less than 16
channels
[ 6.107736] nvme1n1: p1 p2
[ 9.197278] XFS (nvme1n1p1): EXPERIMENTAL reflink feature enabled. Use at your
own risk!
[ 9.198884] XFS (nvme1n1p1): Mounting V5 Filesystem
[ 9.206623] XFS (nvme1n1p1): Ending clean mount
[ 9.214850] XFS (nvme1n1p2): EXPERIMENTAL reflink feature enabled. Use at your
own risk!
[ 9.216365] XFS (nvme1n1p2): Mounting V5 Filesystem
[ 9.225062] XFS (nvme1n1p2): Ending clean mount
[ 9.226362] XFS (nvme1n1p2): Quotacheck needed: Please wait.
[ 9.229295] XFS (nvme1n1p2): Quotacheck: Done.
[ 15.458268] systemd-journald[394]:
/var/log/journal/ec2ee8191318514c081e65b692e6aae3/system.journal: Journal file has
been deleted, rotating.
[ 15.858012] IPv6: ADDRCONF(NETDEV_UP): eth0: link is not ready
[ 16.635401] IPv6: ADDRCONF(NETDEV_CHANGE): eth0: link becomes ready
[ 115.341540] bridge: filtering via arp/ip/ip6tables is no longer available by
default. Update your scripts to load br_netfilter if you need this.
[ 115.346290] Bridge firewalling registered
[ 115.589211] nf_conntrack version 0.5.0 (65536 buckets, 262144 max)
[ 115.958527] Netfilter messages via NETLINK v0.30.
[ 115.962088] ctnetlink v0.93: registering with nfnetlink.
[ 116.492559] IPv6: ADDRCONF(NETDEV_UP): docker0: link is not ready
[ 116.642888] IPv6: ADDRCONF(NETDEV_UP): br-6279bed511c1: link is not ready

# dump instance controller log

tail -n 100 /emr/instance-controller/log/instance-controller.log
at aws157.instancecontroller.rpc.RpcHandler.handle(RpcHandler.java:48)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:333)
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.j
ava:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(
ExecuteProduceConsume.java:303)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(Execute
ProduceConsume.java:148)
 at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceCons
ume.java:136)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.sql.BatchUpdateException: transaction rollback: serialization
failure
at org.hsqldb.jdbc.JDBCPreparedStatement.executeBatch(Unknown Source)
at org.hibernate.jdbc.BatchingBatcher.doExecuteBatch(BatchingBatcher.java:70)
at org.hibernate.jdbc.AbstractBatcher.executeBatch(AbstractBatcher.java:268)
... 24 more
2025-05-12 16:31:17,101 INFO arpc-2074: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:17,118 INFO arpc-1854: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:17,721 INFO instance-metrics-collector-2: Starting Command: [sh
/usr/bin//memmet.sh null]
2025-05-12 16:31:17,724 INFO instance-metrics-collector-2: sh /usr/bin//memmet.sh
null returned 11898
2025-05-12 16:31:17,724 INFO instance-metrics-collector-2: Completed Command:
[sh /usr/bin//memmet.sh null] in 3ms, exitCode: 0, hadExceptionsInExecution?:
false,
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: hdfs-site
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: httpfs-site
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: hadoop-env
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: yarn-env
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: yarn-site
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: mapred-site
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: core-site
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: spark-env
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: spark-defaults
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: hdfs-site
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: httpfs-site
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: hadoop-env
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: yarn-env
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: yarn-site
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: mapred-site
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: core-site
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: spark-env
2025-05-12 16:31:18,551 INFO JobFlowStateUpdater: Masking ComponentConfiguration
for classification: spark-defaults
2025-05-12 16:31:18,553 INFO JobFlowStateUpdater: Sleeping for 120000
2025-05-12 16:31:19,725 INFO instance-metrics-collector-3: Starting Command: [sh
/usr/bin//cpumet.sh null]
2025-05-12 16:31:20,729 INFO instance-metrics-collector-3: sh /usr/bin//cpumet.sh
null returned 0.00
0.13
2025-05-12 16:31:20,729 INFO instance-metrics-collector-3: Completed Command:
[sh /usr/bin//cpumet.sh null] in 1003ms, exitCode: 0, hadExceptionsInExecution?:
false,
2025-05-12 16:31:21,144 INFO arpc-2184: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:21,726 INFO instance-metrics-collector-5: Starting Command: [sh
/usr/bin//diskmet.sh null]
2025-05-12 16:31:22,058 INFO arpc-2075: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:22,107 INFO instance-metrics-collector-5: Completed Command:
[sh /usr/bin//diskmet.sh null] in 381ms, exitCode: 0, hadExceptionsInExecution?:
false,
2025-05-12 16:31:23,726 INFO instance-metrics-collector-1: Starting Command: [sh
/usr/bin//proc_cpu_mem.sh null]
2025-05-12 16:31:25,900 INFO instance-metrics-collector-1: Completed Command:
[sh /usr/bin//proc_cpu_mem.sh null] in 2173ms, exitCode: 0,
hadExceptionsInExecution?: false,
2025-05-12 16:31:26,143 INFO arpc-2184: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:27,058 INFO arpc-1522: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:28,583 INFO RemoteInstanceStateUpdater: Got 2 signed policies from
master
2025-05-12 16:31:28,583 INFO RemoteInstanceStateUpdater: save policy to db:
SignedPolicy SYSTEM_LOG accessKeyId: ASIAXPSMUPSITRMMQRPM secretAccessKey: dUdj****
sessionToken IQoJ**** bucket: aws157-logs-prod prefix: j-3HC44JMM6GERU/ expiration:
2025-05-12 05:21:41.733 PM
2025-05-12 16:31:28,584 INFO RemoteInstanceStateUpdater: save policy to db:
SignedPolicy USER_LOG accessKey: bucket: solagora-lake-rede-hml-emr-logs prefix:
j-3HC44JMM6GERU/ expiration: 2025-05-12 05:21:41.747 PM
2025-05-12 16:31:28,584 INFO RemoteInstanceStateUpdater: Got
doLogPusherDecommissioning = false from master.
2025-05-12 16:31:28,586 INFO RemoteInstanceStateUpdater: Parsing
ComponentConfigurations.
2025-05-12 16:31:28,586 INFO RemoteInstanceStateUpdater: hadoopConfDir
/etc/hadoop/conf/
2025-05-12 16:31:28,586 INFO RemoteInstanceStateUpdater: Apps installed []
2025-05-12 16:31:28,586 INFO RemoteInstanceStateUpdater: Kerberos classification
not found, it is not a Kerberos cluster.
2025-05-12 16:31:28,586 INFO RemoteInstanceStateUpdater: It is not an external KDC
cluster, not setting KerberosReadyToShutDown in common instance record.
2025-05-12 16:31:28,586 INFO RemoteInstanceStateUpdater: Not Starting Kerberos
Refresher, since its not a kerberized cluster
2025-05-12 16:31:28,586 INFO RemoteInstanceStateUpdater:
ClusterConfigurationUpdateHandler Received Managed resize Configuration as null
2025-05-12 16:31:28,586 INFO RemoteInstanceStateUpdater: Touching heartbeat file:
uplink
2025-05-12 16:31:28,586 INFO RemoteInstanceStateUpdater: Sleeping for '60000'
2025-05-12 16:31:31,143 INFO arpc-2184: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:32,058 INFO arpc-752: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:36,143 INFO arpc-2184: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:37,058 INFO arpc-1854: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:41,143 INFO arpc-2184: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:42,058 INFO arpc-1522: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:44,787 INFO MetricsCollectorCoreTaskManager-1: Calling
GetManagedResizeConfigurationForWorkerRequest RPC.
2025-05-12 16:31:44,794 INFO MetricsCollectorCoreTaskManager-1: Received
GetManagedResizeConfigurationForWorkerRequest response, isManagedResizeEnabled is
false.
2025-05-12 16:31:44,794 INFO MetricsCollectorCoreTaskManager-1: Instance level
metrics frequencies are not set, so we will not start MetricCollector on Core/Task
Instances.
2025-05-12 16:31:46,143 INFO arpc-2184: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:47,058 INFO arpc-752: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:48,427 INFO dsm-1: /mnt total 446702 MB free 443061 MB used
3641 MB
2025-05-12 16:31:48,427 INFO dsm-1: / total 10218 MB free 2795 MB used
7423 MB
2025-05-12 16:31:48,427 INFO dsm-1: cycle 41 disk: /mnt freeDiskSpaceMb:
443061/446702 MB freeDiskSpaceRatio:0.99
2025-05-12 16:31:48,427 INFO dsm-1: Disk /mnt had sufficient free space, skipping
directories scan for this cycle
2025-05-12 16:31:48,427 INFO dsm-1: cycle 41 disk: /emr freeDiskSpaceMb: 5039/5110
MB freeDiskSpaceRatio:0.99
2025-05-12 16:31:48,427 INFO dsm-1: Disk /emr had sufficient free space, skipping
directories scan for this cycle
2025-05-12 16:31:51,144 INFO arpc-2184: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:52,058 INFO arpc-2075: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:56,144 INFO arpc-2184: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:31:57,058 INFO arpc-1854: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:32:01,144 INFO arpc-2184: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:32:02,058 INFO arpc-1522: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:32:06,143 INFO arpc-2184: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:32:07,058 INFO arpc-752: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:32:11,144 INFO arpc-1854: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:32:12,058 INFO arpc-2184: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:32:14,787 INFO MetricsCollectorCoreTaskManager-1: Calling
GetManagedResizeConfigurationForWorkerRequest RPC.
2025-05-12 16:32:14,793 INFO MetricsCollectorCoreTaskManager-1: Received
GetManagedResizeConfigurationForWorkerRequest response, isManagedResizeEnabled is
false.
2025-05-12 16:32:14,793 INFO MetricsCollectorCoreTaskManager-1: Instance level
metrics frequencies are not set, so we will not start MetricCollector on Core/Task
Instances.
2025-05-12 16:32:16,143 INFO arpc-2074: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false
2025-05-12 16:32:17,058 INFO arpc-1854: Updating local logpusher decommissioning
state as a result of /getStatus rpc call before: false, after: false

# dump instance controller stdout

tail -n 100 /emr/instance-controller/log/instance-controller.out
tail: cannot open '/emr/instance-controller/log/instance-controller.out' for
reading: No such file or directory

# dump log pusher log

tail -n 100 /emr/logpusher/log/logpusher.log
2025-05-12 16:31:29,103 INFO logspusher-3: Lsof recently identified that file
monitor.log.2025-05-12-15 has no open file handles
2025-05-12 16:31:29,114 INFO logspusher-3: File path is:
/emr/setup-devices/setup_var_cache_dir.log
2025-05-12 16:31:29,114 INFO logspusher-3: Lsof recently identified that file
setup_var_cache_dir.log has no open file handles
2025-05-12 16:31:29,125 INFO logspusher-3: File path is:
/emr/instance-controller/log/bootstrap-actions/1/controller
2025-05-12 16:31:29,125 INFO logspusher-3: Lsof recently identified that file
controller has no open file handles
2025-05-12 16:31:29,141 INFO logspusher-3: File path is:
/emr/instance-state/instance-state.log-2025-05-12-16-30
2025-05-12 16:31:29,141 INFO logspusher-3: Lsof recently identified that file
instance-state.log-2025-05-12-16-30 has open file handles
2025-05-12 16:31:29,152 INFO logspusher-3: File path is: /mnt/var/log/provision-
node/reports/0/f40d2ec6-9646-4704-adcb-cd0a4c7e21bc/ip-10-20-77-92.ec2.internal/
202505121552.puppetreport.json
2025-05-12 16:31:29,152 INFO logspusher-3: Lsof recently identified that file
202505121552.puppetreport.json has no open file handles
2025-05-12 16:31:29,164 INFO logspusher-3: File path is:
/emr/setup-devices/setup_drives.log
2025-05-12 16:31:29,164 INFO logspusher-3: Lsof recently identified that file
setup_drives.log has no open file handles
2025-05-12 16:31:29,175 INFO logspusher-3: File path is:
/emr/apppusher/log/apppusher.log
2025-05-12 16:31:29,175 INFO logspusher-3: Lsof recently identified that file
apppusher.log has open file handles
2025-05-12 16:31:29,187 INFO logspusher-3: File path is:
/emr/instance-state/instance-state.log-2025-05-12-16-15
2025-05-12 16:31:29,187 INFO logspusher-3: Lsof recently identified that file
instance-state.log-2025-05-12-16-15 has no open file handles
2025-05-12 16:31:29,199 INFO logspusher-3: File path is:
/mnt/var/log/hadoop-hdfs/hadoop-hdfs-datanode-ip-10-20-77-92.log.2025-05-12-15
2025-05-12 16:31:29,199 INFO logspusher-3: Lsof recently identified that file
hadoop-hdfs-datanode-ip-10-20-77-92.log.2025-05-12-15 has no open file handles
2025-05-12 16:31:29,210 INFO logspusher-3: File path is:
/mnt/var/log/hadoop-hdfs/hadoop-hdfs-datanode-ip-10-20-77-92.out
2025-05-12 16:31:29,210 INFO logspusher-3: Lsof recently identified that file
hadoop-hdfs-datanode-ip-10-20-77-92.out has open file handles
2025-05-12 16:31:29,223 INFO logspusher-3: File path is:
/emr/setup-devices/setup_var_lib_dir.log
2025-05-12 16:31:29,223 INFO logspusher-3: Lsof recently identified that file
setup_var_lib_dir.log has no open file handles
2025-05-12 16:31:29,223 INFO logspusher-3: Found 47 logs, can't read 0, to upload
5, directory scan time 9ms, dbQuery time 7ms
2025-05-12 16:31:29,223 INFO logspusher-3: Following 5 logs will be uploaded:
age: 87(s) size: 54 /emr/instance-state/instance-state.log-2025-05-12-16-
30
SYS: lastupload: never
USE: lastupload: never
age: 10(s) size: 6552 /emr/instance-controller/log/monitor.log
SYS: lastupload: 300(s) ago, size: 408
age: 12(s) size: 12240 /emr/apppusher/log/apppusher.log
SYS: lastupload: 300(s) ago, size: 687
age: 0(s) size: 378392 /emr/instance-controller/log/instance-controller.log
SYS: lastupload: 300(s) ago, size: 13326
age: 0(s) size: 220608 /emr/logpusher/log/logpusher.log
SYS: lastupload: 300(s) ago, size: 16994
2025-05-12 16:31:29,223 INFO logspusher-7: Compressed /emr/instance-state/instance-
state.log-2025-05-12-16-30 54:73 ratio:0.7397260273972602
2025-05-12 16:31:29,223 INFO logspusher-7: Finished compressing file instance-
state.log-2025-05-12-16-30.gz with time: 0 milliseconds
2025-05-12 16:31:29,223 INFO logspusher-4: Compressed
/emr/instance-controller/log/monitor.log 6552:467 ratio:14.029978586723768
2025-05-12 16:31:29,224 INFO logspusher-4: Finished compressing file monitor.log.gz
with time: 1 milliseconds
2025-05-12 16:31:29,224 INFO logspusher-5: Compressed
/emr/apppusher/log/apppusher.log 12240:777 ratio:15.752895752895753
2025-05-12 16:31:29,224 INFO logspusher-5: Finished compressing file
apppusher.log.gz with time: 1 milliseconds
2025-05-12 16:31:29,224 INFO logspusher-7: Start uploading file: instance-
state.log-2025-05-12-16-30.gz, path: /mnt/var/log/tmp/emr/instance-state/instance-
state.log-2025-05-12-16-30.gz with diff upload, bucketName: aws157-logs-prod,
keyName: j-3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/instance-state/instance-
state.log-2025-05-12-16-30.gz, partSize: 5242880
2025-05-12 16:31:29,224 INFO logspusher-4: Start uploading file: monitor.log.gz,
path: /mnt/var/log/tmp/emr/instance-controller/log/monitor.log.gz with diff upload,
bucketName: aws157-logs-prod, keyName:
j-3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/monitor.log.gz, partSize: 5242880
2025-05-12 16:31:29,224 INFO logspusher-5: Start uploading file: apppusher.log.gz,
path: /mnt/var/log/tmp/emr/apppusher/log/apppusher.log.gz with diff upload,
bucketName: aws157-logs-prod, keyName:
j-3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/apppusher.log.gz, partSize:
5242880
2025-05-12 16:31:29,227 INFO logspusher-1: Compressed
/emr/logpusher/log/logpusher.log 222989:19038 ratio:11.71283748292888
2025-05-12 16:31:29,227 INFO logspusher-1: Finished compressing file
logpusher.log.gz with time: 4 milliseconds
2025-05-12 16:31:29,228 INFO logspusher-1: Start uploading file: logpusher.log.gz,
path: /mnt/var/log/tmp/emr/logpusher/log/logpusher.log.gz with diff upload,
bucketName: aws157-logs-prod, keyName:
j-3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/logpusher.log.gz, partSize:
5242880
2025-05-12 16:31:29,228 INFO logspusher-2: Compressed
/emr/instance-controller/log/instance-controller.log 378392:15617
ratio:24.229493500672344
2025-05-12 16:31:29,228 INFO logspusher-2: Finished compressing file instance-
controller.log.gz with time: 5 milliseconds
2025-05-12 16:31:29,228 INFO logspusher-2: Start uploading file: instance-
controller.log.gz, path: /mnt/var/log/tmp/emr/instance-controller/log/instance-
controller.log.gz with diff upload, bucketName: aws157-logs-prod, keyName: j-
3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/instance-controller.log.gz,
partSize: 5242880
2025-05-12 16:31:29,260 INFO logspusher-1: Compressed File in S3 is smaller than
5MB. Uploading the entire file again. FileName:logpusher.log.gz, Size on S3:
16994 , Size on Disk:19038, in Bucket is /emr/logpusher/log/logpusher.log
2025-05-12 16:31:29,260 INFO logspusher-7: Compressed File in S3 is smaller than
5MB. Uploading the entire file again. FileName:instance-state.log-2025-05-12-16-
30.gz, Size on S3: 0 , Size on Disk:73, in Bucket is /emr/instance-state/instance-
state.log-2025-05-12-16-30
2025-05-12 16:31:29,260 INFO logspusher-5: Compressed File in S3 is smaller than
5MB. Uploading the entire file again. FileName:apppusher.log.gz, Size on S3: 687 ,
Size on Disk:777, in Bucket is /emr/apppusher/log/apppusher.log
2025-05-12 16:31:29,265 INFO logspusher-2: Compressed File in S3 is smaller than
5MB. Uploading the entire file again. FileName:instance-controller.log.gz, Size on
S3: 13326 , Size on Disk:15617, in Bucket is /emr/instance-controller/log/instance-
controller.log
2025-05-12 16:31:29,270 INFO logspusher-4: Compressed File in S3 is smaller than
5MB. Uploading the entire file again. FileName:monitor.log.gz, Size on S3: 408 ,
Size on Disk:467, in Bucket is /emr/instance-controller/log/monitor.log
2025-05-12 16:31:29,281 INFO logspusher-5: Successfully executed multi-part upload
for file: j-3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/apppusher.log.gz
2025-05-12 16:31:29,284 INFO logspusher-7: Successfully executed multi-part upload
for file: j-3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/instance-state/instance-
state.log-2025-05-12-16-30.gz
2025-05-12 16:31:29,287 INFO logspusher-2: Successfully executed multi-part upload
for file: j-3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/instance-
controller.log.gz
2025-05-12 16:31:29,297 INFO logspusher-4: Successfully executed multi-part upload
for file: j-3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/monitor.log.gz
2025-05-12 16:31:29,362 INFO logspusher-2: LogPile to update, s3Path:
/emr/instance-controller/log/instance-controller.log, current S3 size: 13326, new
S3 size: 15617
2025-05-12 16:31:29,371 INFO logspusher-1: Successfully executed multi-part upload
for file: j-3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/logpusher.log.gz
2025-05-12 16:31:29,374 INFO logspusher-7: LogPile to update, s3Path:
/emr/instance-state/instance-state.log-2025-05-12-16-30, current S3 size: null, new
S3 size: 73
2025-05-12 16:31:29,374 INFO logspusher-7: Start uploading file: instance-
state.log-2025-05-12-16-30.gz, path: /mnt/var/log/tmp/emr/instance-state/instance-
state.log-2025-05-12-16-30.gz with diff upload, bucketName: solagora-lake-rede-hml-
emr-logs, keyName: j-3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/instance-
state/instance-state.log-2025-05-12-16-30.gz, partSize: 5242880
2025-05-12 16:31:29,375 INFO logspusher-4: LogPile to update, s3Path:
/emr/instance-controller/log/monitor.log, current S3 size: 408, new S3 size: 467
2025-05-12 16:31:29,385 INFO logspusher-5: LogPile to update, s3Path:
/emr/apppusher/log/apppusher.log, current S3 size: 687, new S3 size: 777
2025-05-12 16:31:29,419 INFO logspusher-7: Compressed File in S3 is smaller than
5MB. Uploading the entire file again. FileName:instance-state.log-2025-05-12-16-
30.gz, Size on S3: 0 , Size on Disk:73, in Bucket is /emr/instance-state/instance-
state.log-2025-05-12-16-30
2025-05-12 16:31:29,439 INFO logspusher-7: Successfully executed multi-part upload
for file: j-3HC44JMM6GERU/node/i-061e4f936a6a69c05/daemons/instance-state/instance-
state.log-2025-05-12-16-30.gz
2025-05-12 16:31:29,449 INFO logspusher-1: LogPile to update, s3Path:
/emr/logpusher/log/logpusher.log, current S3 size: 16994, new S3 size: 19038
2025-05-12 16:31:29,521 INFO logspusher-7: LogPile to update, s3Path:
/emr/instance-state/instance-state.log-2025-05-12-16-30, current S3 size: null, new
S3 size: 73
2025-05-12 16:31:29,522 INFO logspusher-3: Uploaded 5 logs:
prepare: 0(ms) db: 2(ms) size: 73 compress: 0.74 /emr/instance-
state/instance-state.log-2025-05-12-16-30 USE: 146(ms) SYS: 149(ms)
prepare: 0(ms) db: 1(ms) size: 467 compress: 14.03 /emr/instance-
controller/log/monitor.log SYS: 150(ms)
prepare: 0(ms) db: 1(ms) size: 777 compress: 15.75
/emr/apppusher/log/apppusher.log SYS: 160(ms)
prepare: 4(ms) db: 1(ms) size: 15617 compress: 24.23 /emr/instance-
controller/log/instance-controller.log SYS: 133(ms)
prepare: 4(ms) db: 1(ms) size: 19038 compress: 11.71
/emr/logpusher/log/logpusher.log SYS: 220(ms)
2025-05-12 16:31:29,523 INFO logspusher-3: HibernateDb contains 2 S3Policys
2025-05-12 16:31:29,523 INFO logspusher-3: HibernateDb contains 70 LogFiles
2025-05-12 16:31:29,523 INFO logspusher-3: logpusher finished normal iteration (9)
in (1) seconds.
2025-05-12 16:31:29,523 INFO logspusher-3: Acquired Lock for CHECKPOINT DEFRAG
2025-05-12 16:31:29,524 INFO logspusher-3: Checkpoint start
2025-05-12 16:31:29,524 INFO logspusher-3: defrag start
2025-05-12 16:31:29,526 INFO logspusher-3: dataFileCache commit start
2025-05-12 16:31:29,526 INFO logspusher-3: dataFileCache commit end
2025-05-12 16:31:29,526 INFO logspusher-3: dataFileCache commit start
2025-05-12 16:31:29,527 INFO logspusher-3: dataFileCache commit end
2025-05-12 16:31:29,528 INFO logspusher-3: dataFileCache open start
2025-05-12 16:31:29,528 INFO logspusher-3: dataFileCache open end
2025-05-12 16:31:29,529 INFO logspusher-3: defrag end
2025-05-12 16:31:29,529 INFO logspusher-3: Checkpoint end - txts: 1189
2025-05-12 16:31:29,529 INFO logspusher-3: Defrag checkpoint Statement Finished in
6 milliseconds
2025-05-12 16:31:31,144 INFO logspusher-8: Got doLogPusherDecommissioning = false
from local IC DB.
2025-05-12 16:31:31,145 INFO logspusher-6: Touching heartbeat file: logpusher
2025-05-12 16:31:36,144 INFO logspusher-9: Got doLogPusherDecommissioning = false
from local IC DB.
2025-05-12 16:31:41,144 INFO logspusher-4: Got doLogPusherDecommissioning = false
from local IC DB.
2025-05-12 16:31:46,144 INFO logspusher-1: Got doLogPusherDecommissioning = false
from local IC DB.
2025-05-12 16:31:51,145 INFO logspusher-3: Got doLogPusherDecommissioning = false
from local IC DB.
2025-05-12 16:31:56,145 INFO logspusher-6: Got doLogPusherDecommissioning = false
from local IC DB.
2025-05-12 16:32:01,144 INFO logspusher-2: Got doLogPusherDecommissioning = false
from local IC DB.
2025-05-12 16:32:06,144 INFO logspusher-5: Got doLogPusherDecommissioning = false
from local IC DB.
2025-05-12 16:32:11,145 INFO logspusher-1: Got doLogPusherDecommissioning = false
from local IC DB.
2025-05-12 16:32:16,144 INFO logspusher-8: Got doLogPusherDecommissioning = false
from local IC DB.

# dump log pusher stdout

tail -n 100 /emr/logpusher/log/logpusher.out

set +o verbose
============= Java processes for user hadoop =============
3946 aws157.instancecontroller.Main -Djava.security.properties=/emr/instance-
controller/lib/security.properties -Xmx1024m -Djute.maxbuffer=3145728 -XX:
+ExitOnOutOfMemoryError -XX:MinHeapFreeRatio=10 -Dlog4j.defaultInitOverride
=========== End java processes for user hadoop ===========
============= Java processes for user hdfs =============
8234 org.apache.hadoop.hdfs.server.datanode.DataNode -Dproc_datanode -
Djava.net.preferIPv4Stack=true -XX:+ExitOnOutOfMemoryError -
Dhadoop.security.logger=ERROR,DRFAS -Dyarn.log.dir=/var/log/hadoop-hdfs -
Dyarn.log.file=hadoop-hdfs-datanode-ip-10-20-77-92.log
-Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console -
Djava.library.path=:/usr/lib/hadoop-lzo/lib/native:/usr/lib/hadoop/lib/native -
Xmx762m -Dhadoop.log.dir=/var/log/hadoop-hdfs -Dhadoop.log.file=hadoop-hdfs-
datanode-ip-10-20-77-92.log -Dhadoop.home.dir=/usr/lib/hadoop -Dhadoop.id.str=hdfs
-Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-policy.xml -
Dsun.net.inetaddr.ttl=3600
=========== End java processes for user hdfs ===========
============= Java processes for user root =============
3879 aws157.apppusher.Main -Xmx1500m -Xms300m -XX:+ExitOnOutOfMemoryError -
XX:MinHeapFreeRatio=10 -Dlog4j.defaultInitOverride
3832 aws157.logpusher.Main -Xmx1024m -XX:+ExitOnOutOfMemoryError -
XX:MinHeapFreeRatio=10 -Dlog4j.defaultInitOverride
3946 aws157.instancecontroller.Main -Djava.security.properties=/emr/instance-
controller/lib/security.properties -Xmx1024m -Djute.maxbuffer=3145728 -XX:
+ExitOnOutOfMemoryError -XX:MinHeapFreeRatio=10 -Dlog4j.defaultInitOverride
8234 org.apache.hadoop.hdfs.server.datanode.DataNode -Dproc_datanode -
Djava.net.preferIPv4Stack=true -XX:+ExitOnOutOfMemoryError -
Dhadoop.security.logger=ERROR,DRFAS -Dyarn.log.dir=/var/log/hadoop-hdfs -
Dyarn.log.file=hadoop-hdfs-datanode-ip-10-20-77-92.log
-Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console -
Djava.library.path=:/usr/lib/hadoop-lzo/lib/native:/usr/lib/hadoop/lib/native -
Xmx762m -Dhadoop.log.dir=/var/log/hadoop-hdfs -Dhadoop.log.file=hadoop-hdfs-
datanode-ip-10-20-77-92.log -Dhadoop.home.dir=/usr/lib/hadoop -Dhadoop.id.str=hdfs
-Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-policy.xml -
Dsun.net.inetaddr.ttl=3600
8572 org.apache.hadoop.yarn.server.nodemanager.NodeManager -Dproc_nodemanager -
Djava.net.preferIPv4Stack=true -XX:+ExitOnOutOfMemoryError -XX:
+ExitOnOutOfMemoryError -Dsun.net.inetaddr.ttl=30 -Dyarn.log.dir=/var/log/hadoop-
yarn -Dyarn.log.file=hadoop-yarn-nodemanager-ip-10-20-77-92.log
-Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console -
Djava.library.path=:/usr/lib/hadoop-lzo/lib/native:/usr/lib/hadoop/lib/native -
Xmx2048m -Dhadoop.log.dir=/var/log/hadoop-yarn -Dhadoop.log.file=hadoop-yarn-
nodemanager-ip-10-20-77-92.log -Dhadoop.home.dir=/usr/lib/hadoop -
Dhadoop.id.str=yarn -Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-
policy.xml -Dhadoop.security.logger=INFO,NullAppender
=========== End java processes for user root ===========
============= Java processes for user yarn =============
8572 org.apache.hadoop.yarn.server.nodemanager.NodeManager -Dproc_nodemanager -
Djava.net.preferIPv4Stack=true -XX:+ExitOnOutOfMemoryError -XX:
+ExitOnOutOfMemoryError -Dsun.net.inetaddr.ttl=30 -Dyarn.log.dir=/var/log/hadoop-
yarn -Dyarn.log.file=hadoop-yarn-nodemanager-ip-10-20-77-92.log
-Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console -
Djava.library.path=:/usr/lib/hadoop-lzo/lib/native:/usr/lib/hadoop/lib/native -
Xmx2048m -Dhadoop.log.dir=/var/log/hadoop-yarn -Dhadoop.log.file=hadoop-yarn-
nodemanager-ip-10-20-77-92.log -Dhadoop.home.dir=/usr/lib/hadoop -
Dhadoop.id.str=yarn -Dhadoop.root.logger=INFO,DRFA -Dhadoop.policy.file=hadoop-
policy.xml -Dhadoop.security.logger=INFO,NullAppender
=========== End java processes for user yarn ===========

============= Begin threads dump for process 3946 of hadoop =============

3946 aws157.instancecontroller.Main
2025-05-12 16:32:17
Full thread dump OpenJDK 64-Bit Server VM (25.442-b06 mixed mode):

"process reaper" #2190 daemon prio=10 os_prio=0 tid=0x0000fffec8001000 nid=0x1e79

waiting on condition [0x0000ffff19b7a000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00000000c00cfcc0> (a
java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.j
ava:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:
362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"process reaper" #2187 daemon prio=10 os_prio=0 tid=0x0000fffe90004800 nid=0x1e76

waiting on condition [0x0000ffff18141000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c00cfcc0> (a
java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.j
ava:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:
362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"arpc-2184" #2184 prio=5 os_prio=0 tid=0x0000fffe88005800 nid=0x1e70 runnable

[0x0000fffeba7fd000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000eea4a608> (a sun.nio.ch.Util$3)
- locked <0x00000000eea4a5f8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000eea4a3a0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:2
43)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:
191)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(
ExecuteProduceConsume.java:249)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(Execute
ProduceConsume.java:148)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceCons
ume.java:136)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:750)

"arpc-2075" #2075 prio=5 os_prio=0 tid=0x0000fffec802b000 nid=0x193b waiting on

condition [0x0000ffff19342000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c0b91b50> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:56
3)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:750)

"arpc-2074" #2074 prio=5 os_prio=0 tid=0x0000fffed800a800 nid=0x1939 runnable

[0x0000fffebb1fd000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000eea4af10> (a sun.nio.ch.Util$3)
- locked <0x00000000eea4b180> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000eea4afe8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:2
43)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:
191)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(
ExecuteProduceConsume.java:249)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(Execute
ProduceConsume.java:148)
 at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceCons
ume.java:136)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:750)

"arpc-2073" #2073 prio=5 os_prio=0 tid=0x0000fffed8001000 nid=0x1938 runnable

[0x0000fffebaffd000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000eea4fa20> (a sun.nio.ch.Util$3)
- locked <0x00000000eea4fc90> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000eea4faf8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:2
43)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:
191)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(
ExecuteProduceConsume.java:249)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(Execute
ProduceConsume.java:148)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceCons
ume.java:136)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:750)

"arpc-1964" #1964 prio=5 os_prio=0 tid=0x0000fffe98005800 nid=0x1425 runnable

[0x0000fffebb3fd000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000eea4b648> (a sun.nio.ch.Util$3)
- locked <0x00000000eea4f530> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000eea4b720> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:2
43)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:
191)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(
ExecuteProduceConsume.java:249)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(Execute
ProduceConsume.java:148)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceCons
ume.java:136)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:750)

"arpc-1854" #1854 prio=5 os_prio=0 tid=0x0000fffe98001000 nid=0xdd4 waiting on

condition [0x0000fffebb9fe000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c0b91b50> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:56
3)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:750)

"arpc-752" #752 prio=5 os_prio=0 tid=0x0000fffeac006800 nid=0x4bb7 waiting on

condition [0x0000ffff19542000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c0b91b50> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:56
3)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:750)

"MetricsCollectorCoreTaskManager-1" #81 daemon prio=5 os_prio=0

tid=0x0000fffee0420800 nid=0x162f waiting on condition [0x0000fffebb7fd000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c0842e08> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"Scheduler-713877640" #78 prio=5 os_prio=0 tid=0x0000fffee4088800 nid=0x1625

waiting on condition [0x0000fffebbbfd000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000eea901a8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"Attach Listener" #76 daemon prio=9 os_prio=0 tid=0x0000ffff4c001000 nid=0x15b5

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"java-sdk-http-connection-reaper" #71 daemon prio=5 os_prio=0

tid=0x0000fffee01f7800 nid=0x10c8 waiting on condition [0x0000ffff19ffe000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)

"dsm-1" #70 daemon prio=5 os_prio=0 tid=0x0000ffff853a6800 nid=0x10c7 waiting on

condition [0x0000ffff1a5fd000]
 java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c0b8f090> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"JobFlowStateUpdater" #67 daemon prio=5 os_prio=0 tid=0x0000fffee011f000 nid=0x10c2

waiting on condition [0x0000ffff18b42000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
aws157.instancecontroller.common.JobFlowStateUpdater.runLoop(JobFlowStateUpdater.ja
va:106)
at
aws157.instancecontroller.common.JobFlowStateUpdater.access$000(JobFlowStateUpdater
.java:60)
at
aws157.instancecontroller.common.JobFlowStateUpdater$1.run(JobFlowStateUpdater.java
:115)

"RemoteInstanceStateUpdater" #66 daemon prio=5 os_prio=0 tid=0x0000fffee0109000

nid=0x10b8 waiting on condition [0x0000ffff18f42000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000eea920c8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2163)
at aws157.instancecontroller.util.PollingThread$1.run(PollingThread.java:66)

"arpc-63-acceptor-0@6d0d569c-ServerConnector@4f761637{HTTP/1.1,[http/1.1]}
{localhost:8321}" #63 prio=3 os_prio=0 tid=0x0000fffecc001000 nid=0x10b5 runnable
[0x0000ffff19142000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249)
- locked <0x00000000eea46ed0> (a java.lang.Object)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:397)
 at
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:750)

"InstanceConfiguratorDoConfigureThread" #50 daemon prio=5 os_prio=0

tid=0x0000fffeec10c000 nid=0x1091 waiting on condition [0x0000ffff19b42000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c06ad278> (a
java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.j
ava:458)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:
362)
at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"ShutdownActionRunner" #35 prio=5 os_prio=0 tid=0x0000ffff853a3000 nid=0x105e

waiting on condition [0x0000ffff1a3fe000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
aws157.instancecontroller.common.ShutdownActionRunner.doRun(ShutdownActionRunner.ja
va:78)
at
aws157.instancecontroller.common.ShutdownActionRunner.access$000(ShutdownActionRunn
er.java:29)
at
aws157.instancecontroller.common.ShutdownActionRunner$1.run(ShutdownActionRunner.ja
va:71)

"MonitorReactionExecutor" #11 prio=5 os_prio=0 tid=0x0000ffff85395000 nid=0x1057

waiting on condition [0x0000ffff1a7fe000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
aws157.instancecontroller.monitor.executor.MonitorReactionExecutor.runLoop(MonitorR
eactionExecutor.java:87)
at
aws157.instancecontroller.monitor.executor.MonitorReactionExecutor$1.run(MonitorRea
ctionExecutor.java:40)

"instance-metrics-collector-5" #32 daemon prio=5 os_prio=0 tid=0x0000ffff85394000

nid=0x1056 waiting on condition [0x0000ffff1a9fd000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00000000c06b0cc0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"instance-metrics-collector-4" #31 daemon prio=5 os_prio=0 tid=0x0000ffff85391800

nid=0x1055 waiting on condition [0x0000ffff1abfd000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c06b0cc0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"instance-metrics-collector-3" #30 daemon prio=5 os_prio=0 tid=0x0000ffff8538f800

nid=0x1053 waiting on condition [0x0000ffff1adfd000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c06b0cc0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"instance-metrics-collector-2" #29 daemon prio=5 os_prio=0 tid=0x0000ffff8538e800

nid=0x1052 waiting on condition [0x0000ffff1b1fd000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c06b0cc0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"instance-metrics-collector-1" #28 daemon prio=5 os_prio=0 tid=0x0000ffff8534d800

nid=0x1051 waiting on condition [0x0000ffff1b3fd000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c06b0cc0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"json-http-client-hbase-1" #20 daemon prio=5 os_prio=0 tid=0x0000ffff85271000

nid=0x103c waiting on condition [0x0000ffff1b5fd000]
 java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c06b2368> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"json-http-client-mr-am-proxy-1" #19 daemon prio=5 os_prio=0 tid=0x0000ffff85267800

nid=0x103b waiting on condition [0x0000ffff1b7fd000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c06b2f38> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"json-http-client-history-server-1" #18 daemon prio=5 os_prio=0

tid=0x0000ffff85261800 nid=0x103a waiting on condition [0x0000ffff1b9fd000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c06b3ac8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
 at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"json-http-client-rm-1" #17 daemon prio=5 os_prio=0 tid=0x0000ffff85260800

nid=0x1039 waiting on condition [0x0000ffff40e58000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c06b4630> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"HSQLDB Timer @7a362b6b" #14 daemon prio=5 os_prio=0 tid=0x0000ffff84f6e800

nid=0xfed in Object.wait() [0x0000ffff40c59000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at org.hsqldb.lib.HsqlTimer$TaskQueue.park(Unknown Source)
- locked <0x00000000c00b8308> (a org.hsqldb.lib.HsqlTimer$TaskQueue)
at org.hsqldb.lib.HsqlTimer.nextTask(Unknown Source)
- locked <0x00000000c00b8308> (a org.hsqldb.lib.HsqlTimer$TaskQueue)
at org.hsqldb.lib.HsqlTimer$TaskRunner.run(Unknown Source)
at java.lang.Thread.run(Thread.java:750)

"Service Thread" #9 daemon prio=9 os_prio=0 tid=0x0000ffff840d9800 nid=0xf86

runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C1 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x0000ffff840d4000 nid=0xf85

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x0000ffff840d2000 nid=0xf84

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x0000ffff840d0000 nid=0xf83

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x0000ffff840cd000 nid=0xf82

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x0000ffff840b7000 nid=0xf81

runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x0000ffff84097800 nid=0xf80 in

Object.wait() [0x0000ffff5d48c000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
- locked <0x00000000c00c89f8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:188)

"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x0000ffff84093000 nid=0xf7f in

Object.wait() [0x0000ffff5d68c000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
- locked <0x00000000c00d7c98> (a java.lang.ref.Reference$Lock)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)

"main" #1 prio=5 os_prio=0 tid=0x0000ffff84011800 nid=0xf70 waiting on condition

[0x0000ffff8ba5f000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at aws157.instancecontroller.Main.main(Main.java:228)

"VM Thread" os_prio=0 tid=0x0000ffff84089000 nid=0xf7e runnable

"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x0000ffff84028000 nid=0xf76 runnable

"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x0000ffff8402a000 nid=0xf77 runnable

"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x0000ffff8402b800 nid=0xf78 runnable

"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x0000ffff8402d800 nid=0xf79 runnable

"GC task thread#4 (ParallelGC)" os_prio=0 tid=0x0000ffff8402f000 nid=0xf7a runnable

"GC task thread#5 (ParallelGC)" os_prio=0 tid=0x0000ffff84031000 nid=0xf7b runnable

"GC task thread#6 (ParallelGC)" os_prio=0 tid=0x0000ffff84032800 nid=0xf7c runnable

"GC task thread#7 (ParallelGC)" os_prio=0 tid=0x0000ffff84034800 nid=0xf7d runnable

"VM Periodic Task Thread" os_prio=0 tid=0x0000ffff840dc000 nid=0xf87 waiting on

condition

JNI global references: 384

============== End threads dump for process 3946 of hadoop ==============

============= Begin threads dump for process 8234 of hdfs =============
8234 org.apache.hadoop.hdfs.server.datanode.DataNode
2025-05-12 16:32:18
Full thread dump OpenJDK 64-Bit Server VM (25.442-b06 mixed mode):

"java.util.concurrent.ThreadPoolExecutor$Worker@4c57583f[State = -1, empty queue]"

#71 daemon prio=5 os_prio=0 tid=0x0000aaab16ab8000 nid=0x64a8 waiting on condition
[0x0000ffff7bc7d000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000f3da9238> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"java.util.concurrent.ThreadPoolExecutor$Worker@11198451[State = -1, empty queue]"

#68 daemon prio=5 os_prio=0 tid=0x0000ffffa024f800 nid=0x2ac3 waiting on condition
[0x0000ffff7b07e000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000f3da8bb0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"pool-4-thread-1" #65 daemon prio=5 os_prio=0 tid=0x0000ffffa105a000 nid=0x2ac2

waiting on condition [0x0000ffff7b27e000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0aba9f0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1081)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"refreshUsed-/mnt/hdfs/current/BP-1946732164-10.20.77.117-1747065184318" #59 daemon

prio=5 os_prio=0 tid=0x0000ffffa132d800 nid=0x2abc waiting on condition
[0x0000ffff7ba7e000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java
:205)
at java.lang.Thread.run(Thread.java:750)

"VolumeScannerThread(/mnt/hdfs)" #56 daemon prio=5 os_prio=0 tid=0x0000ffffa0453800

nid=0x2ab8 in Object.wait() [0x0000ffff7be7e000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000f3d85ff8> (a
org.apache.hadoop.hdfs.server.datanode.VolumeScanner)
at
org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656)
- locked <0x00000000f3d85ff8> (a
org.apache.hadoop.hdfs.server.datanode.VolumeScanner)

"IPC Parameter Sending Thread #0" #55 daemon prio=5 os_prio=0

tid=0x0000ffffa18fe800 nid=0x2ab7 waiting on condition [0x0000ffff7c27d000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0ef4ea8> (a
java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.j
ava:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:
362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
 at java.lang.Thread.run(Thread.java:750)

"IPC Client (68857875) connection to ip-10-20-77-117.ec2.internal/10.20.77.117:8020

from hdfs" #54 daemon prio=5 os_prio=0 tid=0x0000ffffa2c40000 nid=0x2ab6 in
Object.wait() [0x0000ffff7c47e000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1102)
- locked <0x00000000d0e7f7d0> (a org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1149)

"Attach Listener" #52 daemon prio=9 os_prio=0 tid=0x0000ffff90134800 nid=0x28bb

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"IPC Server handler 9 on default port 9867" #50 daemon prio=5 os_prio=0
tid=0x0000ffffa2781000 nid=0x2299 waiting on condition [0x0000ffff7c89f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0f0bd48> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 8 on default port 9867" #49 daemon prio=5 os_prio=0
tid=0x0000ffffa277f800 nid=0x2298 waiting on condition [0x0000ffff7ca9f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0f0bd48> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 7 on default port 9867" #48 daemon prio=5 os_prio=0
tid=0x0000ffffa277d800 nid=0x2297 waiting on condition [0x0000ffff7cc9f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0f0bd48> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)
"IPC Server handler 6 on default port 9867" #47 daemon prio=5 os_prio=0
tid=0x0000ffffa2e7b800 nid=0x2296 waiting on condition [0x0000ffff7ce9f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0f0bd48> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 5 on default port 9867" #46 daemon prio=5 os_prio=0
tid=0x0000ffffa2e79800 nid=0x2295 waiting on condition [0x0000ffff7d09f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0f0bd48> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 4 on default port 9867" #45 daemon prio=5 os_prio=0
tid=0x0000ffffa2e77800 nid=0x2294 waiting on condition [0x0000ffff7d29f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0f0bd48> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 3 on default port 9867" #44 daemon prio=5 os_prio=0
tid=0x0000ffffa2e76000 nid=0x2293 waiting on condition [0x0000ffff7d49f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0f0bd48> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 2 on default port 9867" #43 daemon prio=5 os_prio=0
tid=0x0000ffffa2e74000 nid=0x2292 waiting on condition [0x0000ffff7d69f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0f0bd48> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 1 on default port 9867" #42 daemon prio=5 os_prio=0
tid=0x0000ffffa2ee8000 nid=0x2291 waiting on condition [0x0000ffff7d89f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0f0bd48> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 0 on default port 9867" #41 daemon prio=5 os_prio=0
tid=0x0000ffffa2eea000 nid=0x2290 waiting on condition [0x0000ffff7da9f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0f0bd48> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server listener on 9867" #31 daemon prio=5 os_prio=0 tid=0x0000ffffa2cb6800

nid=0x228f runnable [0x0000ffff7dc9f000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000d0f0e828> (a sun.nio.ch.Util$3)
- locked <0x00000000d0f0e818> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000d0f0e458> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
 at org.apache.hadoop.ipc.Server$NioListener.run(Server.java:1648)

"IPC Server Responder" #34 daemon prio=5 os_prio=0 tid=0x0000ffffa2cb2800

nid=0x228e runnable [0x0000ffff7de9f000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000d0f0f5b8> (a sun.nio.ch.Util$3)
- locked <0x00000000d0f0f5a8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000d0f0f490> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.hadoop.ipc.Server$NioResponder.doRunLoop(Server.java:1794)
at org.apache.hadoop.ipc.Server$NioResponder.run(Server.java:1777)

"org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@68df9280" #21 daemon

prio=5 os_prio=0 tid=0x0000ffffa2cb1800 nid=0x228d runnable [0x0000ffff7e09f000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249)
- locked <0x00000000d0b456f0> (a java.lang.Object)
at sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113)
- locked <0x00000000d0b456c0> (a java.lang.Object)
at org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85)
at
org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java
:229)
at java.lang.Thread.run(Thread.java:750)

"pool-10-thread-1" #40 prio=5 os_prio=0 tid=0x0000ffffa2cb4000 nid=0x228c waiting

on condition [0x0000ffff7e29f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0f1c1c0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"AsyncAppender-Dispatcher-Thread-21" #39 daemon prio=5 os_prio=0

tid=0x0000ffffa2ec1800 nid=0x228b in Object.wait() [0x0000ffff7e49f000]
 java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.log4j.AsyncAppender$Dispatcher.run(AsyncAppender.java:539)
- locked <0x00000000d0eff880> (a java.util.ArrayList)
at java.lang.Thread.run(Thread.java:750)

"BP-1946732164-10.20.77.117-1747065184318 heartbeating to ip-10-20-77-

117.ec2.internal/10.20.77.117:8020" #38 daemon prio=5 os_prio=0
tid=0x0000ffffa2e64800 nid=0x228a in Object.wait() [0x0000ffff7e89f000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at
org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIB
R(IncrementalBlockReportManager.java:158)
- locked <0x00000000d0e85988> (a
org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager)
at
org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.j
ava:739)
at
org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:879)
at java.lang.Thread.run(Thread.java:750)

"Command processor" #37 daemon prio=5 os_prio=0 tid=0x0000ffffa2695800 nid=0x2289

waiting on condition [0x0000ffff7ea9f000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0ee4f28> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.proce
ssQueue(BPServiceActor.java:1331)
at
org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(B
PServiceActor.java:1315)

"IPC Server idle connection scanner for port 9867" #33 daemon prio=5 os_prio=0
tid=0x0000ffffa2554800 nid=0x2285 in Object.wait() [0x0000ffff7ee9f000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000000d0f0dcb8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"Socket Reader #1 for port 9867" #32 prio=5 os_prio=0 tid=0x0000ffffa2552800

nid=0x2284 runnable [0x0000ffff7f09f000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000d0f0ee80> (a sun.nio.ch.Util$3)
 - locked <0x00000000d0f0ee70> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000d0f0ed58> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.apache.hadoop.ipc.Server$NioListener$Reader.doRunLoop(Server.java:1586)
- locked <0x00000000d0f0eb18> (a
org.apache.hadoop.ipc.Server$NioListener$Reader)
at org.apache.hadoop.ipc.Server$NioListener$Reader.run(Server.java:1565)

"org.apache.hadoop.util.JvmPauseMonitor$Monitor@53499d85" #30 daemon prio=5

os_prio=0 tid=0x0000ffffa2494000 nid=0x2283 waiting on condition
[0x0000ffff7f5f5000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
at java.lang.Thread.run(Thread.java:750)

"nioEventLoopGroup-2-1" #29 prio=10 os_prio=0 tid=0x0000ffffa2489800 nid=0x2282

runnable [0x0000ffff7f7f4000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000d0ef6c80> (a
io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x00000000d0ef6c98> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000d0ef6c38> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySet
Selector.java:68)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
at
io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.
java:995)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:3
0)
at java.lang.Thread.run(Thread.java:750)

"Session-HouseKeeper-1c3146bc-1" #28 prio=5 os_prio=0 tid=0x0000ffffa1eaf800

nid=0x2253 waiting on condition [0x0000ffff7fbf5000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0effc30> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
 at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"qtp369671357-27" #27 daemon prio=5 os_prio=0 tid=0x0000ffffa1ec3000 nid=0x2250

waiting on condition [0x0000ffff7fff5000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d09bfae8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.
java:974)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:101
8)
at java.lang.Thread.run(Thread.java:750)

"qtp369671357-26" #26 daemon prio=5 os_prio=0 tid=0x0000ffffa1ec1000 nid=0x224e

waiting on condition [0x0000ffff801f5000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d09bfae8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.
java:974)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:101
8)
at java.lang.Thread.run(Thread.java:750)

"qtp369671357-25-acceptor-0@67ab6221-ServerConnector@4c2cc639{HTTP/1.1, (http/1.1)}
{localhost:38771}" #25 daemon prio=3 os_prio=0 tid=0x0000ffffa1ebf800 nid=0x224d
runnable [0x0000ffff803f5000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249)
- locked <0x00000000d0e5be80> (a java.lang.Object)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:103
4)
at java.lang.Thread.run(Thread.java:750)

"qtp369671357-24" #24 daemon prio=5 os_prio=0 tid=0x0000ffffa1ebe800 nid=0x224c

runnable [0x0000ffff805f4000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000d0e9e858> (a sun.nio.ch.Util$3)
- locked <0x00000000d0e9e848> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000d0e9e720> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:6
06)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:
543)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.ja
va:362)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java
:186)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.jav
a:173)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:1
37)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$78/1110195322.run(Unknown
Source)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:103
4)
at java.lang.Thread.run(Thread.java:750)

"pool-7-thread-1" #23 prio=5 os_prio=0 tid=0x0000ffffa1711000 nid=0x2244 waiting on

condition [0x0000ffff807f5000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000d0fb3040> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"datanode DomainSocketWatcher" #22 daemon prio=5 os_prio=0 tid=0x0000ffffa17e7800

nid=0x2241 runnable [0x0000ffff80bf5000]
java.lang.Thread.State: RUNNABLE
at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method)
at
org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:
52)
at
org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:503)
at java.lang.Thread.run(Thread.java:750)

"SSL Certificates Store Monitor" #18 daemon prio=5 os_prio=0 tid=0x0000ffffa06c7800

nid=0x223f in Object.wait() [0x0000ffff80df5000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000d06c7a08> (a java.util.TaskQueue)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
- locked <0x00000000d06c7a08> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"Timer for 'DataNode' metrics system" #17 daemon prio=5 os_prio=0

tid=0x0000ffffa0d5a800 nid=0x223e in Object.wait() [0x0000ffff80ff5000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000000d06c89b8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"cloudwatch" #16 daemon prio=5 os_prio=0 tid=0x0000ffffa0e5f800 nid=0x223d in

Object.wait() [0x0000ffff813f5000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.hadoop.metrics2.impl.SinkQueue.waitForData(SinkQueue.java:114)
- locked <0x00000000d09191b8> (a org.apache.hadoop.metrics2.impl.SinkQueue)
at org.apache.hadoop.metrics2.impl.SinkQueue.consumeAll(SinkQueue.java:83)
at
org.apache.hadoop.metrics2.impl.MetricsSinkAdapter.publishMetricsFromQueue(MetricsS
inkAdapter.java:135)
at
org.apache.hadoop.metrics2.impl.MetricsSinkAdapter$1.run(MetricsSinkAdapter.java:89
)

"org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #14
daemon prio=5 os_prio=0 tid=0x0000ffffa1b99800 nid=0x2238 in Object.wait()
[0x0000ffff82166000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000d0930f38> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
- locked <0x00000000d0930f38> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
at
org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileS
ystem.java:4089)
at java.lang.Thread.run(Thread.java:750)

"Service Thread" #9 daemon prio=9 os_prio=0 tid=0x0000ffffa00ff000 nid=0x203d

runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C1 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x0000ffffa00e2000 nid=0x203c

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x0000ffffa00df800 nid=0x203b

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x0000ffffa00dd800 nid=0x203a

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x0000ffffa00da800 nid=0x2039

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x0000ffffa00c4800 nid=0x2038

runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x0000ffffa00a5000 nid=0x2037 in

Object.wait() [0x0000ffff8be7a000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000d066acd0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
- locked <0x00000000d066acd0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:188)

"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x0000ffffa009a800 nid=0x2036

in Object.wait() [0x0000ffff8c07a000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000d0939098> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
- locked <0x00000000d0939098> (a java.lang.ref.Reference$Lock)
 at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)

"Listener at 0.0.0.0/9867" #1 prio=5 os_prio=0 tid=0x0000ffffa0019000 nid=0x202c in

Object.wait() [0x0000ffffa5ad4000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000d0e0dff0> (a java.lang.Thread)
at java.lang.Thread.join(Thread.java:1257)
- locked <0x00000000d0e0dff0> (a java.lang.Thread)
at java.lang.Thread.join(Thread.java:1331)
at
org.apache.hadoop.hdfs.server.datanode.BPServiceActor.join(BPServiceActor.java:612)
at
org.apache.hadoop.hdfs.server.datanode.BPOfferService.join(BPOfferService.java:360)
at
org.apache.hadoop.hdfs.server.datanode.BlockPoolManager.joinAll(BlockPoolManager.ja
va:143)
at org.apache.hadoop.hdfs.server.datanode.DataNode.join(DataNode.java:2788)
at
org.apache.hadoop.hdfs.server.datanode.DataNode.secureMain(DataNode.java:2924)
at org.apache.hadoop.hdfs.server.datanode.DataNode.main(DataNode.java:2946)

"VM Thread" os_prio=0 tid=0x0000ffffa0091000 nid=0x2035 runnable

"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x0000ffffa002f800 nid=0x202d

runnable

"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x0000ffffa0031000 nid=0x202e

runnable

"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x0000ffffa0033000 nid=0x202f

runnable

"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x0000ffffa0034800 nid=0x2030

runnable

"GC task thread#4 (ParallelGC)" os_prio=0 tid=0x0000ffffa0036800 nid=0x2031

runnable

"GC task thread#5 (ParallelGC)" os_prio=0 tid=0x0000ffffa0038800 nid=0x2032

runnable

"GC task thread#6 (ParallelGC)" os_prio=0 tid=0x0000ffffa003a000 nid=0x2033

runnable

"GC task thread#7 (ParallelGC)" os_prio=0 tid=0x0000ffffa003c000 nid=0x2034

runnable

"VM Periodic Task Thread" os_prio=0 tid=0x0000ffffa0102000 nid=0x203e waiting on

condition

JNI global references: 426

============== End threads dump for process 8234 of hdfs ==============

============= Begin threads dump for process 3832 of root =============

3832 aws157.logpusher.Main
2025-05-12 16:32:18
Full thread dump OpenJDK 64-Bit Server VM (25.442-b06 mixed mode):

"process reaper" #613 daemon prio=10 os_prio=0 tid=0x0000ffff30047000 nid=0x1f61

waiting on condition [0x0000ffff83302000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c0014fd8> (a
java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.j
ava:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:
362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"Attach Listener" #59 daemon prio=9 os_prio=0 tid=0x0000ffff70001000 nid=0x1298

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"java-sdk-http-connection-reaper" #58 daemon prio=5 os_prio=0

tid=0x0000ffff28025800 nid=0x1079 waiting on condition [0x0000ffff6542e000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188)

"logspusher-9" #57 daemon prio=5 os_prio=0 tid=0x0000ffff38123800 nid=0x1047

waiting on condition [0x0000ffff80778000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c08566c8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"logspusher-8" #56 daemon prio=5 os_prio=0 tid=0x0000ffff38122800 nid=0x1045

waiting on condition [0x0000ffff80b78000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c08566c8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"logspusher-7" #55 daemon prio=5 os_prio=0 tid=0x0000ffff38116800 nid=0x1044

waiting on condition [0x0000ffff80978000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c08566c8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"logspusher-6" #20 daemon prio=5 os_prio=0 tid=0x0000ffff3c00e800 nid=0xfbe waiting

on condition [0x0000ffff80f99000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c08566c8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"logspusher-5" #19 daemon prio=5 os_prio=0 tid=0x0000ffff34001000 nid=0xfbd waiting

on condition [0x0000ffff81199000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c08566c8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"logspusher-4" #18 daemon prio=5 os_prio=0 tid=0x0000ffffad349000 nid=0xfbc waiting

on condition [0x0000ffff81399000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c08566c8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
"logspusher-3" #17 daemon prio=5 os_prio=0 tid=0x0000ffffad348000 nid=0xfbb waiting
on condition [0x0000ffff81599000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c08566c8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"logspusher-2" #16 daemon prio=5 os_prio=0 tid=0x0000ffffad33b800 nid=0xfba waiting

on condition [0x0000ffff81799000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c08566c8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"logspusher-1" #15 daemon prio=5 os_prio=0 tid=0x0000ffffad33a800 nid=0xfb9 waiting

on condition [0x0000ffff81999000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000c08566c8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
 at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"HSQLDB Timer @5f7b97da" #13 daemon prio=5 os_prio=0 tid=0x0000ffffac95b000

nid=0xfb7 in Object.wait() [0x0000ffff821b6000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at org.hsqldb.lib.HsqlTimer$TaskQueue.park(Unknown Source)
- locked <0x00000000c06e83f8> (a org.hsqldb.lib.HsqlTimer$TaskQueue)
at org.hsqldb.lib.HsqlTimer.nextTask(Unknown Source)
- locked <0x00000000c06e83f8> (a org.hsqldb.lib.HsqlTimer$TaskQueue)
at org.hsqldb.lib.HsqlTimer$TaskRunner.run(Unknown Source)
at java.lang.Thread.run(Thread.java:750)

"MultiThreadedHttpConnectionManager cleanup" #12 daemon prio=5 os_prio=0

tid=0x0000ffffaccce000 nid=0xfb3 in Object.wait() [0x0000ffff82eb9000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
- locked <0x00000000c0005630> (a java.lang.ref.ReferenceQueue$Lock)
at
org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$ReferenceQueueThre
ad.run(MultiThreadedHttpConnectionManager.java:1082)

"Service Thread" #9 daemon prio=9 os_prio=0 tid=0x0000ffffac0d9000 nid=0xf42

runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C1 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x0000ffffac0d4000 nid=0xf41

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x0000ffffac0d2000 nid=0xf40

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x0000ffffac0d0000 nid=0xf3e

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x0000ffffac0cd000 nid=0xf35

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x0000ffffac0b7000 nid=0xf34

runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x0000ffffac097800 nid=0xf19 in
Object.wait() [0x0000ffff90aaa000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
- locked <0x00000000c000d7d0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:188)

"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x0000ffffac093000 nid=0xf18 in

Object.wait() [0x0000ffff90caa000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
- locked <0x00000000c0005c58> (a java.lang.ref.Reference$Lock)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)

"main" #1 prio=5 os_prio=0 tid=0x0000ffffac011800 nid=0xf00 waiting on condition

[0x0000ffffb3080000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at aws157.instancecontroller.common.AppProcess.run(AppProcess.java:53)
at aws157.logpusher.Main.main(Main.java:53)

"VM Thread" os_prio=0 tid=0x0000ffffac089000 nid=0xf17 runnable

"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x0000ffffac028000 nid=0xf05 runnable

"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x0000ffffac029800 nid=0xf06 runnable

"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x0000ffffac02b800 nid=0xf07 runnable

"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x0000ffffac02d000 nid=0xf08 runnable

"GC task thread#4 (ParallelGC)" os_prio=0 tid=0x0000ffffac02f000 nid=0xf09 runnable

"GC task thread#5 (ParallelGC)" os_prio=0 tid=0x0000ffffac031000 nid=0xf0a runnable

"GC task thread#6 (ParallelGC)" os_prio=0 tid=0x0000ffffac032800 nid=0xf0c runnable

"GC task thread#7 (ParallelGC)" os_prio=0 tid=0x0000ffffac034800 nid=0xf0d runnable

"VM Periodic Task Thread" os_prio=0 tid=0x0000ffffac0db800 nid=0xf43 waiting on

condition

JNI global references: 327

============== End threads dump for process 3832 of root ==============

============= Begin threads dump for process 3879 of root =============

3879 aws157.apppusher.Main
2025-05-12 16:32:18
Full thread dump OpenJDK 64-Bit Server VM (25.442-b06 mixed mode):

"Attach Listener" #20 daemon prio=9 os_prio=0 tid=0x0000ffff40001000 nid=0x141b

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"pool-2-thread-1" #19 prio=5 os_prio=0 tid=0x0000ffff85ce0000 nid=0x100c waiting on
condition [0x0000ffff25e4d000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e64d06b0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"pool-5-thread-1" #17 prio=5 os_prio=0 tid=0x0000ffff85cde000 nid=0x100b waiting on

condition [0x0000ffff2604d000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e63a0178> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"pool-3-thread-1" #15 prio=5 os_prio=0 tid=0x0000ffff85cdc000 nid=0x100a waiting on

condition [0x0000ffff2624d000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e6398658> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
 at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"Connection evictor" #13 daemon prio=5 os_prio=0 tid=0x0000ffff84ed4800 nid=0xff4

waiting on condition [0x0000ffff2664e000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:
66)
at java.lang.Thread.run(Thread.java:750)

"Log4j2-TF-3-Scheduled-1" #11 daemon prio=5 os_prio=0 tid=0x0000ffff85a78800

nid=0xfcb waiting on condition [0x0000ffff27166000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e63b0178> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"Service Thread" #9 daemon prio=9 os_prio=0 tid=0x0000ffff840e5800 nid=0xf5c

runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C1 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x0000ffff840e0800 nid=0xf5b

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x0000ffff840de800 nid=0xf5a

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x0000ffff840dd000 nid=0xf59
waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x0000ffff840da000 nid=0xf54

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x0000ffff840c4000 nid=0xf52

runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x0000ffff840a4800 nid=0xf4b in

Object.wait() [0x0000ffff60414000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000e6398ea0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
- locked <0x00000000e6398ea0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:188)

"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x0000ffff8409f800 nid=0xf4a in

Object.wait() [0x0000ffff60614000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000e63b8178> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
- locked <0x00000000e63b8178> (a java.lang.ref.Reference$Lock)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)

"main" #1 prio=5 os_prio=0 tid=0x0000ffff8401e800 nid=0xf33 waiting on condition

[0x0000ffff8bc47000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at aws157.apppusher.AppPusherScheduler.sleep(AppPusherScheduler.java:263)
at
aws157.apppusher.AppPusherScheduler.sleepWithClusterShutdownCheck(AppPusherSchedule
r.java:275)
at
aws157.apppusher.AppPusherScheduler.runAsDaemon(AppPusherScheduler.java:210)
at aws157.apppusher.Main.main(Main.java:42)

"VM Thread" os_prio=0 tid=0x0000ffff84096000 nid=0xf48 runnable

"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x0000ffff84034800 nid=0xf36 runnable

"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x0000ffff84036800 nid=0xf37 runnable

"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x0000ffff84038800 nid=0xf38 runnable

"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x0000ffff8403a000 nid=0xf39 runnable

"GC task thread#4 (ParallelGC)" os_prio=0 tid=0x0000ffff8403c000 nid=0xf3a runnable

"GC task thread#5 (ParallelGC)" os_prio=0 tid=0x0000ffff8403d800 nid=0xf3b runnable

"GC task thread#6 (ParallelGC)" os_prio=0 tid=0x0000ffff8403f800 nid=0xf3c runnable

"GC task thread#7 (ParallelGC)" os_prio=0 tid=0x0000ffff84041000 nid=0xf3d runnable

"VM Periodic Task Thread" os_prio=0 tid=0x0000ffff840e8000 nid=0xf5d waiting on

condition

JNI global references: 287

============== End threads dump for process 3879 of root ==============

============= Begin threads dump for process 8572 of yarn =============

8572 org.apache.hadoop.yarn.server.nodemanager.NodeManager
2025-05-12 16:32:19
Full thread dump OpenJDK 64-Bit Server VM (25.442-b06 mixed mode):

"DestroyJavaVM" #165 prio=5 os_prio=0 tid=0x0000ffffb801a000 nid=0x217e waiting on

condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"Node Status Updater" #164 prio=5 os_prio=0 tid=0x0000ffffba4d4000 nid=0x2cd5 in

Object.wait() [0x0000ffff81bf7000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at
org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl$StatusUpdaterRunnab
le.run(NodeStatusUpdaterImpl.java:1458)
- locked <0x00000000800bb550> (a java.lang.Object)
at java.lang.Thread.run(Thread.java:750)

"IPC Parameter Sending Thread #0" #163 daemon prio=5 os_prio=0

tid=0x0000ffffb9d31800 nid=0x2cd4 waiting on condition [0x0000ffff81df7000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000e4fdd4f0> (a
java.util.concurrent.SynchronousQueue$TransferStack)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.j
ava:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:
362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"IPC Client (1317746045) connection to

ip-10-20-77-117.ec2.internal/10.20.77.117:8025 from yarn" #162 daemon prio=5
os_prio=0 tid=0x0000ffffb9d37800 nid=0x2cd3 in Object.wait() [0x0000ffff81ff7000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1102)
- locked <0x00000000e4f85188> (a org.apache.hadoop.ipc.Client$Connection)
 at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1149)

"Attach Listener" #160 daemon prio=9 os_prio=0 tid=0x0000ffffa0131800 nid=0x2a31

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"org.apache.hadoop.util.JvmPauseMonitor$Monitor@420a8042" #158 daemon prio=5

os_prio=0 tid=0x0000ffffbb83e000 nid=0x247d waiting on condition
[0x0000ffff8261b000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
at java.lang.Thread.run(Thread.java:750)

"NM Event dispatcher" #157 prio=5 os_prio=0 tid=0x0000ffffbba5f000 nid=0x247c

waiting on condition [0x0000ffff8281b000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000800bb738> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:140)
at java.lang.Thread.run(Thread.java:750)

"pool-13-thread-1" #156 prio=5 os_prio=0 tid=0x0000ffffba624800 nid=0x247b waiting

on condition [0x0000ffff82e49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000081156290> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"Session-HouseKeeper-714f3da4-1" #155 prio=5 os_prio=0 tid=0x0000ffffba618000

nid=0x247a waiting on condition [0x0000ffff83049000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x0000000081134a50> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"qtp1997753864-154" #154 daemon prio=5 os_prio=0 tid=0x0000ffffb9e00000 nid=0x2479

waiting on condition [0x0000ffff83249000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000080d220f0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.
java:974)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:101
8)
at java.lang.Thread.run(Thread.java:750)

"qtp1997753864-153" #153 daemon prio=5 os_prio=0 tid=0x0000ffffb9372800 nid=0x2478

waiting on condition [0x0000ffff83449000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000080d220f0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.
java:974)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:101
8)
at java.lang.Thread.run(Thread.java:750)
"qtp1997753864-152" #152 daemon prio=5 os_prio=0 tid=0x0000ffffb9370800 nid=0x2477
waiting on condition [0x0000ffff83649000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000080d220f0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.
java:974)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:101
8)
at java.lang.Thread.run(Thread.java:750)

"qtp1997753864-151-acceptor-0@affad1e-ServerConnector@3bd3d05e{HTTP/1.1,
(http/1.1)}{0.0.0.0:8042}" #151 daemon prio=3 os_prio=0 tid=0x0000ffffb936f000
nid=0x2476 runnable [0x0000ffff83849000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249)
- locked <0x0000000081207800> (a java.lang.Object)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:103
4)
at java.lang.Thread.run(Thread.java:750)

"qtp1997753864-150" #150 daemon prio=5 os_prio=0 tid=0x0000ffffb936d000 nid=0x2475

runnable [0x0000ffff83a48000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000e4f0d7f8> (a sun.nio.ch.Util$3)
- locked <0x00000000e4f0d7e8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000e4f0d6c0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:6
06)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:
543)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.ja
va:362)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java
:186)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.jav
a:173)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:1
37)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$99/2108044736.run(Unknown
Source)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:103
4)
at java.lang.Thread.run(Thread.java:750)

"qtp1997753864-149" #149 daemon prio=5 os_prio=0 tid=0x0000ffffb83a2800 nid=0x2474

runnable [0x0000ffff83c48000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000e4f0e088> (a sun.nio.ch.Util$3)
- locked <0x00000000e4f0e078> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000e4f0df50> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:6
06)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:
543)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.ja
va:362)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java
:186)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.jav
a:173)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:1
37)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$99/2108044736.run(Unknown
Source)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
 at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:103
4)
at java.lang.Thread.run(Thread.java:750)

"qtp1997753864-148" #148 daemon prio=5 os_prio=0 tid=0x0000ffffb83a0800 nid=0x2473

runnable [0x0000ffff83e48000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000e4f0e918> (a sun.nio.ch.Util$3)
- locked <0x00000000e4f0e908> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000e4f0e7e0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:6
06)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:
543)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.ja
va:362)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java
:186)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.jav
a:173)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:1
37)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$99/2108044736.run(Unknown
Source)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:103
4)
at java.lang.Thread.run(Thread.java:750)

"qtp1997753864-147" #147 daemon prio=5 os_prio=0 tid=0x0000ffffb839f800 nid=0x2472

runnable [0x0000ffff84048000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000e4f0f1a8> (a sun.nio.ch.Util$3)
- locked <0x00000000e4f0f198> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000e4f0f070> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
 at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:6
06)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:
543)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.ja
va:362)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java
:186)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.jav
a:173)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:1
37)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$99/2108044736.run(Unknown
Source)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:103
4)
at java.lang.Thread.run(Thread.java:750)

"pool-12-thread-1" #146 prio=5 os_prio=0 tid=0x0000ffffbadcb000 nid=0x2471 waiting

on condition [0x0000ffff84649000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000811339d0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"IPC Server handler 63 on default port 8041" #145 daemon prio=5 os_prio=0
tid=0x0000ffffbaccd000 nid=0x2457 waiting on condition [0x0000ffff84a49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 62 on default port 8041" #144 daemon prio=5 os_prio=0
tid=0x0000ffffbaccb000 nid=0x2456 waiting on condition [0x0000ffff84c49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 61 on default port 8041" #143 daemon prio=5 os_prio=0
tid=0x0000ffffbacc9000 nid=0x2455 waiting on condition [0x0000ffff84e49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 60 on default port 8041" #142 daemon prio=5 os_prio=0
tid=0x0000ffffbacc7000 nid=0x2454 waiting on condition [0x0000ffff85049000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 59 on default port 8041" #141 daemon prio=5 os_prio=0
tid=0x0000ffffbacc4800 nid=0x2453 waiting on condition [0x0000ffff85249000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 58 on default port 8041" #140 daemon prio=5 os_prio=0
tid=0x0000ffffbacc2800 nid=0x2452 waiting on condition [0x0000ffff85449000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 57 on default port 8041" #139 daemon prio=5 os_prio=0
tid=0x0000ffffbacc0800 nid=0x2451 waiting on condition [0x0000ffff85649000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 56 on default port 8041" #138 daemon prio=5 os_prio=0
tid=0x0000ffffbacbe800 nid=0x2450 waiting on condition [0x0000ffff85849000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 55 on default port 8041" #137 daemon prio=5 os_prio=0
tid=0x0000ffffbacbc800 nid=0x244f waiting on condition [0x0000ffff85a49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 54 on default port 8041" #136 daemon prio=5 os_prio=0
tid=0x0000ffffbacba000 nid=0x244e waiting on condition [0x0000ffff85c49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 53 on default port 8041" #135 daemon prio=5 os_prio=0
tid=0x0000ffffbacb8000 nid=0x244d waiting on condition [0x0000ffff85e49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 52 on default port 8041" #134 daemon prio=5 os_prio=0
tid=0x0000ffffbacb6000 nid=0x244c waiting on condition [0x0000ffff86049000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 51 on default port 8041" #133 daemon prio=5 os_prio=0
tid=0x0000ffffbacb4000 nid=0x244b waiting on condition [0x0000ffff86249000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 50 on default port 8041" #132 daemon prio=5 os_prio=0
tid=0x0000ffffbacb2000 nid=0x244a waiting on condition [0x0000ffff86449000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 49 on default port 8041" #131 daemon prio=5 os_prio=0
tid=0x0000ffffbacb0000 nid=0x2449 waiting on condition [0x0000ffff86649000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 48 on default port 8041" #130 daemon prio=5 os_prio=0
tid=0x0000ffffbacad800 nid=0x2448 waiting on condition [0x0000ffff86849000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 47 on default port 8041" #129 daemon prio=5 os_prio=0
tid=0x0000ffffbacab800 nid=0x2447 waiting on condition [0x0000ffff86a49000]
java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 46 on default port 8041" #128 daemon prio=5 os_prio=0
tid=0x0000ffffbaca9800 nid=0x2446 waiting on condition [0x0000ffff86c49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 45 on default port 8041" #127 daemon prio=5 os_prio=0
tid=0x0000ffffbaca7800 nid=0x2445 waiting on condition [0x0000ffff86e49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 44 on default port 8041" #126 daemon prio=5 os_prio=0
tid=0x0000ffffbaca5800 nid=0x2444 waiting on condition [0x0000ffff87049000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 43 on default port 8041" #125 daemon prio=5 os_prio=0
tid=0x0000ffffbaca3800 nid=0x2443 waiting on condition [0x0000ffff87249000]
 java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 42 on default port 8041" #124 daemon prio=5 os_prio=0
tid=0x0000ffffbaca1800 nid=0x2442 waiting on condition [0x0000ffff87449000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 41 on default port 8041" #123 daemon prio=5 os_prio=0
tid=0x0000ffffbac9f800 nid=0x2441 waiting on condition [0x0000ffff87649000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 40 on default port 8041" #122 daemon prio=5 os_prio=0
tid=0x0000ffffbac9d800 nid=0x2440 waiting on condition [0x0000ffff87849000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 39 on default port 8041" #121 daemon prio=5 os_prio=0
tid=0x0000ffffbac9b000 nid=0x243f waiting on condition [0x0000ffff87a49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 38 on default port 8041" #120 daemon prio=5 os_prio=0
tid=0x0000ffffbac99000 nid=0x243e waiting on condition [0x0000ffff87c49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 37 on default port 8041" #119 daemon prio=5 os_prio=0
tid=0x0000ffffbac97000 nid=0x243d waiting on condition [0x0000ffff87e49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 36 on default port 8041" #118 daemon prio=5 os_prio=0
tid=0x0000ffffbac95000 nid=0x243c waiting on condition [0x0000ffff88049000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)
"IPC Server handler 35 on default port 8041" #117 daemon prio=5 os_prio=0
tid=0x0000ffffbac92800 nid=0x243b waiting on condition [0x0000ffff88249000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 34 on default port 8041" #116 daemon prio=5 os_prio=0
tid=0x0000ffffbac91000 nid=0x243a waiting on condition [0x0000ffff88449000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 33 on default port 8041" #115 daemon prio=5 os_prio=0
tid=0x0000ffffbac8f000 nid=0x2439 waiting on condition [0x0000ffff88649000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 32 on default port 8041" #114 daemon prio=5 os_prio=0
tid=0x0000ffffbac8d000 nid=0x2438 waiting on condition [0x0000ffff88849000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)
"IPC Server handler 31 on default port 8041" #113 daemon prio=5 os_prio=0
tid=0x0000ffffbac8a800 nid=0x2437 waiting on condition [0x0000ffff88a49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 30 on default port 8041" #112 daemon prio=5 os_prio=0
tid=0x0000ffffbac89000 nid=0x2436 waiting on condition [0x0000ffff88c49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 29 on default port 8041" #111 daemon prio=5 os_prio=0
tid=0x0000ffffbac87000 nid=0x2435 waiting on condition [0x0000ffff88e49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 28 on default port 8041" #110 daemon prio=5 os_prio=0
tid=0x0000ffffbac84800 nid=0x2434 waiting on condition [0x0000ffff89049000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 27 on default port 8041" #109 daemon prio=5 os_prio=0
tid=0x0000ffffbac82800 nid=0x2433 waiting on condition [0x0000ffff89249000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 26 on default port 8041" #108 daemon prio=5 os_prio=0
tid=0x0000ffffbac80800 nid=0x2432 waiting on condition [0x0000ffff89449000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 25 on default port 8041" #107 daemon prio=5 os_prio=0
tid=0x0000ffffbac7e800 nid=0x2431 waiting on condition [0x0000ffff89649000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 24 on default port 8041" #106 daemon prio=5 os_prio=0
tid=0x0000ffffbac7c800 nid=0x2430 waiting on condition [0x0000ffff89849000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 23 on default port 8041" #105 daemon prio=5 os_prio=0
tid=0x0000ffffbac7a000 nid=0x242f waiting on condition [0x0000ffff89a49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 22 on default port 8041" #104 daemon prio=5 os_prio=0
tid=0x0000ffffbac78000 nid=0x242e waiting on condition [0x0000ffff89c49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 21 on default port 8041" #103 daemon prio=5 os_prio=0
tid=0x0000ffffbac76000 nid=0x242d waiting on condition [0x0000ffff89e49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 20 on default port 8041" #102 daemon prio=5 os_prio=0
tid=0x0000ffffbac74000 nid=0x242c waiting on condition [0x0000ffff8a049000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 19 on default port 8041" #101 daemon prio=5 os_prio=0
tid=0x0000ffffbac72000 nid=0x242b waiting on condition [0x0000ffff8a249000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 18 on default port 8041" #100 daemon prio=5 os_prio=0
tid=0x0000ffffbac70000 nid=0x242a waiting on condition [0x0000ffff8a449000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 17 on default port 8041" #99 daemon prio=5 os_prio=0
tid=0x0000ffffbac6e000 nid=0x2429 waiting on condition [0x0000ffff8a649000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 16 on default port 8041" #98 daemon prio=5 os_prio=0
tid=0x0000ffffbac6b800 nid=0x2428 waiting on condition [0x0000ffff8a849000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
 at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 15 on default port 8041" #97 daemon prio=5 os_prio=0
tid=0x0000ffffbac69800 nid=0x2427 waiting on condition [0x0000ffff8aa49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 14 on default port 8041" #96 daemon prio=5 os_prio=0
tid=0x0000ffffbac67800 nid=0x2426 waiting on condition [0x0000ffff8ac49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 13 on default port 8041" #95 daemon prio=5 os_prio=0
tid=0x0000ffffbac65800 nid=0x2425 waiting on condition [0x0000ffff8ae49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 12 on default port 8041" #94 daemon prio=5 os_prio=0
tid=0x0000ffffbac63800 nid=0x2424 waiting on condition [0x0000ffff8b049000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 11 on default port 8041" #93 daemon prio=5 os_prio=0
tid=0x0000ffffbac61800 nid=0x2423 waiting on condition [0x0000ffff8b249000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 10 on default port 8041" #92 daemon prio=5 os_prio=0
tid=0x0000ffffbac5f000 nid=0x2422 waiting on condition [0x0000ffff8b449000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 9 on default port 8041" #91 daemon prio=5 os_prio=0
tid=0x0000ffffbac5d000 nid=0x2421 waiting on condition [0x0000ffff8b649000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 8 on default port 8041" #90 daemon prio=5 os_prio=0
tid=0x0000ffffbac5b000 nid=0x2420 waiting on condition [0x0000ffff8b849000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 7 on default port 8041" #89 daemon prio=5 os_prio=0
tid=0x0000ffffbac59000 nid=0x241f waiting on condition [0x0000ffff8ba49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 6 on default port 8041" #88 daemon prio=5 os_prio=0
tid=0x0000ffffbac57000 nid=0x241e waiting on condition [0x0000ffff8bc49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 5 on default port 8041" #87 daemon prio=5 os_prio=0
tid=0x0000ffffbac55000 nid=0x241d waiting on condition [0x0000ffff8be49000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 4 on default port 8041" #86 daemon prio=5 os_prio=0
tid=0x0000ffffbac53000 nid=0x241c waiting on condition [0x0000ffff8c049000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 3 on default port 8041" #85 daemon prio=5 os_prio=0
tid=0x0000ffffbac50800 nid=0x241b waiting on condition [0x0000ffff8c249000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 2 on default port 8041" #84 daemon prio=5 os_prio=0
tid=0x0000ffffbac4e800 nid=0x241a waiting on condition [0x0000ffff8c449000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 1 on default port 8041" #83 daemon prio=5 os_prio=0
tid=0x0000ffffbac4c800 nid=0x2419 waiting on condition [0x0000ffff8c649000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 0 on default port 8041" #82 daemon prio=5 os_prio=0
tid=0x0000ffffbac4b800 nid=0x2418 waiting on condition [0x0000ffff8c849000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809f2478> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server listener on 8041" #50 daemon prio=5 os_prio=0 tid=0x0000ffffbac48000

nid=0x2417 runnable [0x0000ffff8ca49000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000080ad4bb0> (a sun.nio.ch.Util$3)
- locked <0x0000000080ad4ba0> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000080ad47e0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.apache.hadoop.ipc.Server$NioListener.run(Server.java:1648)

"IPC Server Responder" #53 daemon prio=5 os_prio=0 tid=0x0000ffffbac46800

nid=0x2416 runnable [0x0000ffff8cc49000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000080aa5ba8> (a sun.nio.ch.Util$3)
- locked <0x0000000080aa5bb8> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000080aa5b60> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.hadoop.ipc.Server$NioResponder.doRunLoop(Server.java:1794)
at org.apache.hadoop.ipc.Server$NioResponder.run(Server.java:1777)

"NM ContainerManager dispatcher" #81 prio=5 os_prio=0 tid=0x0000ffffbac44800

nid=0x2415 waiting on condition [0x0000ffff8ce49000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000800b68f0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:140)
at java.lang.Thread.run(Thread.java:750)

"Container Monitor" #18 prio=5 os_prio=0 tid=0x0000ffffbac4a800 nid=0x2414 waiting

on condition [0x0000ffff8d049000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonito
rImpl$MonitoringThread.run(ContainersMonitorImpl.java:603)

"Public Localizer" #28 prio=5 os_prio=0 tid=0x0000ffffbac2e000 nid=0x2413 waiting

on condition [0x0000ffff8d269000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000008076bcc0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ExecutorCompletionService.take(ExecutorCompletionService.java:
193)
at
org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocali
zationService$PublicLocalizer.run(ResourceLocalizationService.java:978)

"IPC Server handler 19 on default port 8040" #79 daemon prio=5 os_prio=0
tid=0x0000ffffbac2c000 nid=0x2412 waiting on condition [0x0000ffff8d469000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 18 on default port 8040" #78 daemon prio=5 os_prio=0
tid=0x0000ffffbac2a000 nid=0x2411 waiting on condition [0x0000ffff8d669000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 17 on default port 8040" #77 daemon prio=5 os_prio=0
tid=0x0000ffffbac28800 nid=0x2410 waiting on condition [0x0000ffff8d869000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 16 on default port 8040" #76 daemon prio=5 os_prio=0
tid=0x0000ffffbac26000 nid=0x240f waiting on condition [0x0000ffff8da69000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 15 on default port 8040" #75 daemon prio=5 os_prio=0
tid=0x0000ffffbac24000 nid=0x240e waiting on condition [0x0000ffff8dc69000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 14 on default port 8040" #74 daemon prio=5 os_prio=0
tid=0x0000ffffbac22800 nid=0x240d waiting on condition [0x0000ffff8de69000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 13 on default port 8040" #73 daemon prio=5 os_prio=0
tid=0x0000ffffbac20000 nid=0x240c waiting on condition [0x0000ffff8e069000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 12 on default port 8040" #72 daemon prio=5 os_prio=0
tid=0x0000ffffbac1e000 nid=0x240b waiting on condition [0x0000ffff8e269000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 11 on default port 8040" #71 daemon prio=5 os_prio=0
tid=0x0000ffffbac1c000 nid=0x240a waiting on condition [0x0000ffff8e469000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 10 on default port 8040" #70 daemon prio=5 os_prio=0
tid=0x0000ffffbac1a000 nid=0x2409 waiting on condition [0x0000ffff8e669000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 9 on default port 8040" #69 daemon prio=5 os_prio=0
tid=0x0000ffffbac18000 nid=0x2408 waiting on condition [0x0000ffff8e869000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 8 on default port 8040" #68 daemon prio=5 os_prio=0
tid=0x0000ffffbac16000 nid=0x2407 waiting on condition [0x0000ffff8ea69000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 7 on default port 8040" #67 daemon prio=5 os_prio=0
tid=0x0000ffffbac14000 nid=0x2406 waiting on condition [0x0000ffff8ec69000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 6 on default port 8040" #66 daemon prio=5 os_prio=0
tid=0x0000ffffbac12800 nid=0x2405 waiting on condition [0x0000ffff8ee69000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 5 on default port 8040" #65 daemon prio=5 os_prio=0
tid=0x0000ffffbac10800 nid=0x2404 waiting on condition [0x0000ffff8f069000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 4 on default port 8040" #64 daemon prio=5 os_prio=0
tid=0x0000ffffbac0e800 nid=0x2403 waiting on condition [0x0000ffff8f269000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 3 on default port 8040" #63 daemon prio=5 os_prio=0
tid=0x0000ffffbac0d000 nid=0x2402 waiting on condition [0x0000ffff8f469000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 2 on default port 8040" #62 daemon prio=5 os_prio=0
tid=0x0000ffffbac0b000 nid=0x2401 waiting on condition [0x0000ffff8f669000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 1 on default port 8040" #61 daemon prio=5 os_prio=0
tid=0x0000ffffbac09800 nid=0x2400 waiting on condition [0x0000ffff8f869000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server handler 0 on default port 8040" #60 daemon prio=5 os_prio=0
tid=0x0000ffffbab9d000 nid=0x23ff waiting on condition [0x0000ffff8fa69000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000809b5308> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3282)

"IPC Server listener on 8040" #56 daemon prio=5 os_prio=0 tid=0x0000ffffbab9c800

nid=0x23fe runnable [0x0000ffff8fc69000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000080a2a488> (a sun.nio.ch.Util$3)
- locked <0x0000000080a2a478> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000080a2a0b8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.apache.hadoop.ipc.Server$NioListener.run(Server.java:1648)

"IPC Server Responder" #59 daemon prio=5 os_prio=0 tid=0x0000ffffbab9b800

nid=0x23fd runnable [0x0000ffff8fe69000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000080a2b230> (a sun.nio.ch.Util$3)
- locked <0x0000000080a2b220> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000080a2b108> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.hadoop.ipc.Server$NioResponder.doRunLoop(Server.java:1794)
at org.apache.hadoop.ipc.Server$NioResponder.run(Server.java:1777)

"IPC Server idle connection scanner for port 8040" #58 daemon prio=5 os_prio=0
tid=0x0000ffffbaba3000 nid=0x23fc in Object.wait() [0x0000ffff90069000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
 - locked <0x0000000080a29908> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"Socket Reader #1 for port 8040" #57 prio=5 os_prio=0 tid=0x0000ffffbaba2000

nid=0x23fb runnable [0x0000ffff90269000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000080a2aaf0> (a sun.nio.ch.Util$3)
- locked <0x0000000080a2aae0> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000080a2a9c8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.apache.hadoop.ipc.Server$NioListener$Reader.doRunLoop(Server.java:1586)
- locked <0x0000000080a2a780> (a
org.apache.hadoop.ipc.Server$NioListener$Reader)
at org.apache.hadoop.ipc.Server$NioListener$Reader.run(Server.java:1565)

"ResourceLocalizationService Cache Cleanup" #55 prio=5 os_prio=0

tid=0x0000ffffbab94000 nid=0x23fa waiting on condition [0x0000ffff90469000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000800b7970> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"IPC Server idle connection scanner for port 8041" #52 daemon prio=5 os_prio=0
tid=0x0000ffffbaa98800 nid=0x23c8 in Object.wait() [0x0000ffff90669000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x0000000080ad4030> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"Socket Reader #1 for port 8041" #51 prio=5 os_prio=0 tid=0x0000ffffbaa96800

nid=0x23c4 runnable [0x0000ffff90869000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000080ad5218> (a sun.nio.ch.Util$3)
- locked <0x0000000080ad5208> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000080ad50f0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.apache.hadoop.ipc.Server$NioListener$Reader.doRunLoop(Server.java:1586)
- locked <0x0000000080ad4ea8> (a
org.apache.hadoop.ipc.Server$NioListener$Reader)
at org.apache.hadoop.ipc.Server$NioListener$Reader.run(Server.java:1565)

"Node Resource Monitor" #17 daemon prio=5 os_prio=0 tid=0x0000ffffbaa23800

nid=0x2349 waiting on condition [0x0000ffff90bca000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitorImpl$MonitoringThread.
run(NodeResourceMonitorImpl.java:205)

"DistributedNodeDescriptorsRunner-Timer" #49 daemon prio=5 os_prio=0

tid=0x0000ffffbaa1d800 nid=0x2348 in Object.wait() [0x0000ffff90dca000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x0000000080a1ed50> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"HealthReporterService-Timer" #48 daemon prio=5 os_prio=0 tid=0x0000ffffba9cf800

nid=0x229c in Object.wait() [0x0000ffff911ca000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x0000000080aa60b8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"DiskHealthMonitor-Timer" #47 daemon prio=5 os_prio=0 tid=0x0000ffffba9ce000

nid=0x229b in Object.wait() [0x0000ffff913ca000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x0000000080a1f180> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"shuffle-boss-2-1" #46 daemon prio=5 os_prio=0 tid=0x0000ffffba9ad000 nid=0x229a

runnable [0x0000ffff915c9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000080ae6918> (a
org.sparkproject.io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x0000000080a1f450> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000080ae6880> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.sparkproject.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(Select
edSelectionKeySetSelector.java:68)
at
org.sparkproject.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813)
at
org.sparkproject.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
at
org.sparkproject.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThr
eadEventExecutor.java:986)
at
org.sparkproject.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.j
ava:74)
at
org.sparkproject.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLoc
alRunnable.java:30)
at java.lang.Thread.run(Thread.java:750)

"New I/O server boss #17" #45 prio=5 os_prio=0 tid=0x0000ffffba328000 nid=0x2281
runnable [0x0000ffff91de9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000008077b3b0> (a sun.nio.ch.Util$3)
- locked <0x000000008077b3c0> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000008077b368> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #16" #44 prio=5 os_prio=0 tid=0x0000ffffba315800 nid=0x2280

runnable [0x0000ffff91fe9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000807515f8> (a sun.nio.ch.Util$3)
- locked <0x0000000080751608> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000807515b0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
 at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #15" #43 prio=5 os_prio=0 tid=0x0000ffffba2e3000 nid=0x227f

runnable [0x0000ffff921e9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000807da288> (a sun.nio.ch.Util$3)
- locked <0x00000000807da278> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000807da160> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #14" #42 prio=5 os_prio=0 tid=0x0000ffffba2af800 nid=0x227e

runnable [0x0000ffff923e9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000080751cb0> (a sun.nio.ch.Util$3)
- locked <0x0000000080751cc0> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000080751c68> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #13" #41 prio=5 os_prio=0 tid=0x0000ffffba27d000 nid=0x227d

runnable [0x0000ffff925e9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000807d9c48> (a sun.nio.ch.Util$3)
- locked <0x00000000807d9c38> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000807d9b20> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
"New I/O worker #12" #40 prio=5 os_prio=0 tid=0x0000ffffba24a000 nid=0x227c
runnable [0x0000ffff927e9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000807d9608> (a sun.nio.ch.Util$3)
- locked <0x00000000807d95f8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000807d94e0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #11" #39 prio=5 os_prio=0 tid=0x0000ffffba217800 nid=0x227b

runnable [0x0000ffff929e9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000080833a88> (a sun.nio.ch.Util$3)
- locked <0x0000000080833a98> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000080833a40> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #10" #38 prio=5 os_prio=0 tid=0x0000ffffba1e4800 nid=0x227a

runnable [0x0000ffff92be9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000008077e768> (a sun.nio.ch.Util$3)
- locked <0x000000008077e758> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000008077e640> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #9" #37 prio=5 os_prio=0 tid=0x0000ffffba1b1800 nid=0x2279 runnable
[0x0000ffff92de9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000008084e998> (a sun.nio.ch.Util$3)
- locked <0x000000008084e9a8> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000008084e950> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #8" #36 prio=5 os_prio=0 tid=0x0000ffffba17f000 nid=0x2278 runnable
[0x0000ffff92fe9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000008077dae8> (a sun.nio.ch.Util$3)
- locked <0x000000008077dad8> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000008077d9c0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #7" #35 prio=5 os_prio=0 tid=0x0000ffffba14c000 nid=0x2277 runnable
[0x0000ffff931e9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000008077d4a8> (a sun.nio.ch.Util$3)
- locked <0x000000008077d498> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000008077d380> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #6" #34 prio=5 os_prio=0 tid=0x0000ffffba023000 nid=0x2276 runnable
[0x0000ffff933e9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000807d8fc8> (a sun.nio.ch.Util$3)
- locked <0x00000000807d8fb8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000807d8ea0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #5" #33 prio=5 os_prio=0 tid=0x0000ffffba021800 nid=0x2275 runnable
[0x0000ffff935e9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000008084f348> (a sun.nio.ch.Util$3)
- locked <0x000000008084f358> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000008084f300> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #4" #32 prio=5 os_prio=0 tid=0x0000ffffba01e800 nid=0x2274 runnable
[0x0000ffff937e9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000008077ce68> (a sun.nio.ch.Util$3)
- locked <0x000000008077ce58> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000008077cd40> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #3" #31 prio=5 os_prio=0 tid=0x0000ffffba01c800 nid=0x2273 runnable
[0x0000ffff939e9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000080888728> (a sun.nio.ch.Util$3)
- locked <0x0000000080888738> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000808886e0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #2" #30 prio=5 os_prio=0 tid=0x0000ffffba01f800 nid=0x2272 runnable
[0x0000ffff93be9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000008077e128> (a sun.nio.ch.Util$3)
- locked <0x000000008077e118> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000008077e000> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"New I/O worker #1" #29 prio=5 os_prio=0 tid=0x0000ffffba01b800 nid=0x2271 runnable
[0x0000ffff93de9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000000807d8988> (a sun.nio.ch.Util$3)
- locked <0x00000000807d8978> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000000807d8860> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.j
ava:434)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java
:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42
)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)

"Timer for 'NodeManager' metrics system" #22 daemon prio=5 os_prio=0

tid=0x0000ffffb9f8b800 nid=0x2243 in Object.wait() [0x0000ffff93fea000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x0000000080874d88> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"cloudwatch" #21 daemon prio=5 os_prio=0 tid=0x0000ffffb9f3a800 nid=0x2242 in

Object.wait() [0x0000ffff943ea000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.hadoop.metrics2.impl.SinkQueue.waitForData(SinkQueue.java:114)
- locked <0x000000008075bde0> (a org.apache.hadoop.metrics2.impl.SinkQueue)
at org.apache.hadoop.metrics2.impl.SinkQueue.consumeAll(SinkQueue.java:83)
 at
org.apache.hadoop.metrics2.impl.MetricsSinkAdapter.publishMetricsFromQueue(MetricsS
inkAdapter.java:135)
at
org.apache.hadoop.metrics2.impl.MetricsSinkAdapter$1.run(MetricsSinkAdapter.java:89
)

"Timer-0" #20 prio=5 os_prio=0 tid=0x0000ffffb85d7000 nid=0x2240 in Object.wait()

[0x0000ffff949ea000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x0000000080004cb8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"NMLeveldbStateStoreService compaction timer" #16 daemon prio=5 os_prio=0

tid=0x0000ffffb811e800 nid=0x223c in Object.wait() [0x0000ffff9503c000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x000000008001c680> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x000000008001c680> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #15
daemon prio=5 os_prio=0 tid=0x0000ffffb9c59000 nid=0x2239 in Object.wait()
[0x0000ffff95995000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x0000000080004e48> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
- locked <0x0000000080004e48> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
at
org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileS
ystem.java:4089)
at java.lang.Thread.run(Thread.java:750)

"Service Thread" #9 daemon prio=9 os_prio=0 tid=0x0000ffffb8104800 nid=0x218f

runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C1 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x0000ffffb80de000 nid=0x218e

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x0000ffffb80dc000 nid=0x218d

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x0000ffffb80d9800 nid=0x218c

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x0000ffffb80d6000 nid=0x218b

waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x0000ffffb80c0800 nid=0x218a

runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE

"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x0000ffffb80a0800 nid=0x2189 in

Object.wait() [0x0000ffff9f7f9000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x000000008001cb68> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
- locked <0x000000008001cb68> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:188)

"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x0000ffffb809c000 nid=0x2188

in Object.wait() [0x0000ffff9f9f9000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000000800053c0> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
- locked <0x00000000800053c0> (a java.lang.ref.Reference$Lock)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)

"VM Thread" os_prio=0 tid=0x0000ffffb8092000 nid=0x2187 runnable

"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x0000ffffb8030800 nid=0x217f

runnable

"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x0000ffffb8032000 nid=0x2180

runnable

"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x0000ffffb8034000 nid=0x2181

runnable

"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x0000ffffb8035800 nid=0x2182

runnable

"GC task thread#4 (ParallelGC)" os_prio=0 tid=0x0000ffffb8037800 nid=0x2183

runnable

"GC task thread#5 (ParallelGC)" os_prio=0 tid=0x0000ffffb8039800 nid=0x2184

runnable

"GC task thread#6 (ParallelGC)" os_prio=0 tid=0x0000ffffb803b000 nid=0x2185

runnable

"GC task thread#7 (ParallelGC)" os_prio=0 tid=0x0000ffffb803d000 nid=0x2186

runnable

"VM Periodic Task Thread" os_prio=0 tid=0x0000ffffb8107800 nid=0x2190 waiting on

condition

JNI global references: 463

============== End threads dump for process 8572 of yarn ==============

# dump service nanny log

tail -n 20 /emr/service-nanny/log/service-nanny-`date +%Y-%m-%d`
date +%Y-%m-%d
tail: cannot open '/emr/service-nanny/log/service-nanny-2025-05-12' for reading: No
such file or directory

# dump mysql process list

mysqladmin -u root processlist | awk -F'|' '{print $6, $7, $8, $9}' | grep -v '^\
s*$'
#mysqladmin: connect to server at 'localhost' failed
error: 'Can't connect to local MySQL server through socket
'/var/lib/mysql/mysql.sock' (2)'
Check that mysqld is running and that the socket: '/var/lib/mysql/mysql.sock'
exists!

# Can we contact dom0?

curl --connect-timeout 2 -q -f --retry 5 -X PUT
"http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds:
1"
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0100
56 100 56 0 0 34250 0 --:--:-- --:--:-- --:--:-- 56000
AQAEACJBsxstHTO1_rieH3GhROJgTFygH1VsB0ZkLakPPHDn2trY6Q==
# Can we access master?
set +o verbose
PING 10.20.77.117 (10.20.77.117) 56(84) bytes of data.
64 bytes from 10.20.77.117: icmp_seq=1 ttl=255 time=0.155 ms
64 bytes from 10.20.77.117: icmp_seq=2 ttl=255 time=0.123 ms

--- 10.20.77.117 ping statistics ---

2 packets transmitted, 2 received, 0% packet loss, time 1000ms
rtt min/avg/max/mdev = 0.123/0.139/0.155/0.016 ms

traceroute to 10.20.77.117 (10.20.77.117), 10 hops max, 60 byte packets

1 10.20.77.117 0.131 ms 0.074 ms 0.071 ms

# Can we access the outside world?

curl -s -I --connect-timeout 1 --max-time 5
http://elasticmapreduce.s3.amazonaws.com | head
HTTP/1.1 200 OK
x-amz-id-2:
wa3qArXoAsnkCg0sMNV7pnX/nMvN0D9zEmw9+MHA97UAK0lTsxgY0oSvhAzHxGgZVE3qG27Q3GU=
x-amz-request-id: RGG3J53BSSDSME05
Date: Mon, 12 May 2025 16:32:21 GMT
x-amz-bucket-region: us-east-1
x-amz-access-point-alias: false
Content-Type: application/xml
Transfer-Encoding: chunked
Server: AmazonS3

# Now traceroute it
traceroute -T --sport=17241 -p 443 -w 3 -n -m 10 elasticmapreduce.s3.amazonaws.com
traceroute to elasticmapreduce.s3.amazonaws.com (52.217.204.25), 10 hops max, 60
byte packets
1 * * *
2 240.0.56.97 0.808 ms 0.755 ms 0.808 ms
3 240.0.56.121 0.771 ms 0.736 ms 0.729 ms
4 240.0.56.57 0.815 ms 0.846 ms 0.796 ms
 5 * * *
6 52.217.204.25 0.760 ms * *

# listing of last logged in users

last -w -n 25
reboot system boot 4.14.355-276.618.amzn2.aarch64 Mon May 12 15:50 - 16:32
(00:42)

wtmp begins Mon May 12 15:50:23 2025

# whats io usage look like

iostat -x 1 5
Linux 4.14.355-276.618.amzn2.aarch64 (ip-10-20-77-92) 05/12/25 _aarch64_
(8 CPU)

avg-cpu: %user %nice %system %iowait %steal %idle

1.01 0.00 0.25 0.18 0.00 98.56

Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz

await r_await w_await svctm %util
nvme1n1 0.00 4.40 0.20 8.60 4.71 182520.48 41447.37
0.01 0.94 0.08 0.96 0.05 0.05
nvme0n1 0.03 2.61 11.61 2.88 266.31 423.66 95.25 0.01
1.86 1.73 2.35 0.34 0.49

avg-cpu: %user %nice %system %iowait %steal %idle

0.00 0.00 0.13 0.00 0.00 99.87

Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz

await r_await w_await svctm %util
nvme1n1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
0.00 0.00 0.00 0.00 0.00
nvme0n1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
0.00 0.00 0.00 0.00 0.00

avg-cpu: %user %nice %system %iowait %steal %idle

0.00 0.00 0.25 0.00 0.00 99.75

Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz

await r_await w_await svctm %util
nvme1n1 0.00 0.00 0.00 2.00 0.00 6.00 6.00 0.00
0.00 0.00 0.00 0.00 0.00
nvme0n1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
0.00 0.00 0.00 0.00 0.00

avg-cpu: %user %nice %system %iowait %steal %idle

1.38 0.00 0.38 0.00 0.00 98.25

Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz

await r_await w_await svctm %util
nvme1n1 0.00 0.00 0.00 2.00 0.00 6.00 6.00 0.00
0.00 0.00 0.00 0.00 0.00
nvme0n1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
0.00 0.00 0.00 0.00 0.00

avg-cpu: %user %nice %system %iowait %steal %idle

0.12 0.00 0.12 0.00 0.00 99.75

Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz

await r_await w_await svctm %util
nvme1n1 0.00 1.00 0.00 93.00 0.00 772.00 16.60 0.00
0.00 0.00 0.00 0.00 0.00
nvme0n1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
0.00 0.00 0.00 0.00 0.00

# whats memory usage look like

free -m
total used free shared buff/cache available
Mem: 15809 1671 11898 0 2240 13825
Swap: 0 0 0

# trend memory
vmstat 1 5
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
0 0 0 12183860 5512 2288444 0 0 34 109 138 208 1 0 99 0 0
0 0 0 12183736 5512 2288444 0 0 0 0 591 847 0 0 99 0 0
0 0 0 12183364 5512 2288448 0 0 0 0 598 920 0 0 100 0
0
0 0 0 12183100 5512 2288464 0 0 0 7 662 1027 0 0 100 0
0
0 0 0 12182832 5512 2288632 0 0 0 5 744 968 0 0 100 0
0

# amount of disk free

df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 7.7G 0 7.7G 0% /dev
tmpfs 7.8G 0 7.8G 0% /dev/shm
tmpfs 7.8G 496K 7.8G 1% /run
tmpfs 7.8G 0 7.8G 0% /sys/fs/cgroup
/dev/nvme0n1p1 10G 7.3G 2.8G 73% /
/dev/nvme0n1p128 10M 3.8M 6.2M 38% /boot/efi
/dev/nvme1n1p1 5.0G 71M 5.0G 2% /emr
/dev/nvme1n1p2 437G 3.6G 433G 1% /mnt
tmpfs 1.6G 0 1.6G 0% /run/user/0

# amount of inode free

df -ih
Filesystem Inodes IUsed IFree IUse% Mounted on
devtmpfs 2.0M 306 2.0M 1% /dev
tmpfs 2.0M 2 2.0M 1% /dev/shm
tmpfs 2.0M 465 2.0M 1% /run
tmpfs 2.0M 16 2.0M 1% /sys/fs/cgroup
/dev/nvme0n1p1 5.0M 117K 4.9M 3% /
/dev/nvme0n1p128 0 0 0 - /boot/efi
/dev/nvme1n1p1 2.5M 86 2.5M 1% /emr
/dev/nvme1n1p2 219M 12K 219M 1% /mnt
tmpfs 2.0M 1 2.0M 1% /run/user/0

set +o verbose
Top 10 folders in /emr in MB:
2 /emr/instance-state
1 /emr/setup-devices
1 /emr/logpusher/run
1 /emr/logpusher/log
1 /emr/logpusher/db
1 /emr/instance-controller/run
1 /emr/instance-controller/log/bootstrap-actions/1
1 /emr/instance-controller/log
1 /emr/instance-controller/lib/sslKeys
1 /emr/instance-controller/lib/info

Top 20 folders including subdirectories in /emr in MB:

3 /emr
2 /emr/instance-controller
2 /emr/instance-state
1 /emr/apppusher
1 /emr/apppusher/log
1 /emr/apppusher/run
1 /emr/instance-controller/db
1 /emr/instance-controller/lib
1 /emr/instance-controller/lib/bootstrap-actions
1 /emr/instance-controller/lib/bootstrap-actions/1
1 /emr/instance-controller/lib/info
1 /emr/instance-controller/lib/sslKeys
1 /emr/instance-controller/log
1 /emr/instance-controller/log/bootstrap-actions
1 /emr/instance-controller/log/bootstrap-actions/1
1 /emr/instance-controller/run
1 /emr/logpusher
1 /emr/logpusher/db
1 /emr/logpusher/log
1 /emr/logpusher/run

# DNS Resolution Details

cat /etc/nsswitch.conf | grep 'hosts:';
#hosts: db files nisplus nis dns
hosts: files dns myhostname

cat /etc/resolv.conf;
options timeout:2 attempts:5
; generated by /usr/sbin/dhclient-script
search ec2.internal
nameserver 10.20.0.2

cat /etc/hosts;
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost6 localhost6.localdomain6

hostname -f
ip-10-20-77-92.ec2.internal

HOSTNAME=$(hostname -f);

(time dig $(hostname -f) +noadditional +noquestion +nocomments +noclass +nocmd) |&
grep -v cmd | grep -v -e '^$'
ip-10-20-77-92.ec2.internal. 36 A 10.20.77.92
;; Query time: 0 msec
;; SERVER: 10.20.0.2#53(10.20.0.2)
;; WHEN: Mon May 12 16:32:52 UTC 2025
;; MSG SIZE rcvd: 72
real 0m0.007s
user 0m0.007s
sys 0m0.000s
(time getent hosts $HOSTNAME) |& grep -v -e '^$';
10.20.77.92 ip-10-20-77-92.ec2.internal
real 0m0.003s
user 0m0.001s
sys 0m0.000s

MYIP=$(getent hosts $HOSTNAME | cut -d ' ' -f 1);

(time dig -x $MYIP +noadditional +noquestion +nocomments +noclass +nocmd) |& grep -
v cmd | grep -v -e '^$'
92.77.20.10.in-addr.arpa. 156 PTR ip-10-20-77-92.ec2.internal.
;; Query time: 0 msec
;; SERVER: 10.20.0.2#53(10.20.0.2)
;; WHEN: Mon May 12 16:32:52 UTC 2025
;; MSG SIZE rcvd: 94
real 0m0.006s
user 0m0.006s
sys 0m0.000s

(time getent hosts $MYIP) |& grep -v -e '^$';

10.20.77.92 ip-10-20-77-92.ec2.internal
real 0m0.001s
user 0m0.001s
sys 0m0.000s

# Top 10 largest files and folders in /mnt/var/*

du -hsx /mnt/var/* | sort -rh | head -10
367M /mnt/var/cache
101M /mnt/var/lib
19M /mnt/var/log
0 /mnt/var/tmp
0 /mnt/var/run
0 /mnt/var/em

# Top 20 largest files and folders including subdirectories in /mnt/var/log

du -hx /mnt/var/log | sort -rh | head -20
19M /mnt/var/log
17M /mnt/var/log/journal/ec2ee8191318514c081e65b692e6aae3
17M /mnt/var/log/journal
680K /mnt/var/log/provision-node
512K /mnt/var/log/provision-node/reports/0/f40d2ec6-9646-4704-adcb-cd0a4c7e21bc/
ip-10-20-77-92.ec2.internal
512K /mnt/var/log/provision-node/reports/0/f40d2ec6-9646-4704-adcb-cd0a4c7e21bc
512K /mnt/var/log/provision-node/reports/0
512K /mnt/var/log/provision-node/reports
348K /mnt/var/log/tmp
228K /mnt/var/log/tmp/emr
160K /mnt/var/log/provision-node/apps-phase/0/f40d2ec6-9646-4704-adcb-cd0a4c7e21bc
160K /mnt/var/log/provision-node/apps-phase/0
160K /mnt/var/log/provision-node/apps-phase
100K /mnt/var/log/chrony
88K /mnt/var/log/tmp/mnt/var/log
88K /mnt/var/log/tmp/mnt/var
88K /mnt/var/log/tmp/mnt
72K /mnt/var/log/tmp/emr/instance-state
72K /mnt/var/log/hadoop-yarn
64K /mnt/var/log/tmp/mnt/var/log/provision-node
# dump network statistics
netstat -s -e
Ip:
Forwarding: 1
57249 total packets received
3 with invalid addresses
0 forwarded
0 incoming packets discarded
57246 incoming packets delivered
45671 requests sent out
Icmp:
27 ICMP messages received
21 input ICMP message failed
ICMP input histogram:
timeout in transit: 21
echo replies: 6
6 ICMP messages sent
0 ICMP messages failed
ICMP output histogram:
echo requests: 6
IcmpMsg:
InType0: 6
InType11: 21
OutType8: 6
Tcp:
961 active connection openings
531 passive connection openings
50 failed connection attempts
9 connection resets received
12 connections established
56448 segments received
45214 segments sent out
0 segments retransmitted
0 bad segments received
372 resets sent
Udp:
771 packets received
0 packets to unknown port received
0 packet receive errors
771 packets sent
0 receive buffer errors
0 send buffer errors
UdpLite:
TcpExt:
550 TCP sockets finished time wait in fast timer
74 delayed acks sent
Quick ack mode was activated 65 times
46608 packet headers predicted
3013 acknowledgments not containing data payload received
10923 predicted acknowledgments
TCPDSACKOldSent: 65
143 connections reset due to unexpected data
73 connections reset due to early user close
TCPRcvCoalesce: 15303
TCPAutoCorking: 13
TCPOrigDataSent: 15728
TCPHystartTrainDetect: 3
TCPHystartTrainCwnd: 60
IpExt:
 InOctets: 473387602
OutOctets: 8725449
InNoECTPkts: 235194

# dump instance metrics

if [ "$dumpType" == "full" ]
then
mon-tool --local_metrics
fi
{
"Instance-Disk-Used-/sys/fs/cgroup": {
"name": "Instance-Disk-Used-/sys/fs/cgroup",
"value": 0.0,
"unit": "Percent",
"time": 1747067482108
},
"Customer-Memory-Top10": {
"name": "Customer-Memory-Top10",
"value": 2.3000000000000003,
"unit": "Percent",
"time": 1747067545901,
"subMetrics": [
{
"name": "/usr/bin/java -Xmx1500m -Xms300m -XX:+ExitOnOutOfMemoryError -
XX:MinHeapFreeRatio\u003d10 -cp
/usr/share/aws/emr/emr-log-analytics-metrics/lib/*:/home/hadoop/conf -
Dlog4j.defaultInitOverride aws157.apppusher.Main-Memory",
"value": 1.3,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "/usr/bin/dockerd -H fd://
--containerd\u003d/run/containerd/containerd.sock --default-ulimit nofile\
u003d32768:65536-Memory",
"value": 0.5,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "/usr/bin/containerd-Memory",
"value": 0.3,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "/usr/lib/systemd/systemd-journald-Memory",
"value": 0.1,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "/usr/bin/amazon-ssm-agent-Memory",
"value": 0.1,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "top-Memory",
 "value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "/usr/lib/systemd/systemd --switched-root --system --deserialize
21-Memory",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "[kthreadd]-Memory",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "[kworker/0:0H]-Memory",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "[kworker/u16:0]-Memory",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
}
]
},
"Instance-CPU-IOWait": {
"name": "Instance-CPU-IOWait",
"value": 0.0,
"unit": "Percent",
"time": 1747067540729
},
"EMR-Memory": {
"name": "EMR-Memory",
"value": 8.0,
"unit": "Percent",
"time": 1747067545901
},
"InstanceController-Memory": {
"name": "InstanceController-Memory",
"value": 1.9,
"unit": "Percent",
"time": 1747067545901
},
"LogPusher-CPU": {
"name": "LogPusher-CPU",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
"LogPusher-Total-S3Policy": {
"name": "LogPusher-Total-S3Policy",
"value": 2.0,
"unit": "Count",
 "time": 1747067326139
},
"Instance-Disk-Used-/run/user/0": {
"name": "Instance-Disk-Used-/run/user/0",
"value": 0.0,
"unit": "Percent",
"time": 1747067482108
},
"YarnNodeManager-Memory": {
"name": "YarnNodeManager-Memory",
"value": 2.9,
"unit": "Percent",
"time": 1747067545901
},
"YarnNodeManager-Reboots": {
"name": "YarnNodeManager-Reboots",
"value": 0.0,
"unit": "Count",
"time": 1747065205901
},
"Instance-Disk-Used-/mnt": {
"name": "Instance-Disk-Used-/mnt",
"value": 1.0,
"unit": "Percent",
"time": 1747067482108,
"subMetrics": [
{
"name": "/mnt/var/cache/yum/aarch64/2/amzn2-core/gen",
"value": 264.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/mnt/var/cache/yum/aarch64/2/amzn2-core",
"value": 66.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/mnt/var/lib/rpm",
"value": 52.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/mnt/var/log/journal/ec2ee8191318514c081e65b692e6aae3",
"value": 17.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/mnt/var/cache/yum/aarch64/2/amzn2-core-source/gen",
"value": 12.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/mnt/var/cache/yum/aarch64/2/amzn2-core-debuginfo/gen",
"value": 10.0,
 "unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/mnt/var/cache/amzn2extras",
"value": 6.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/mnt/var/lib/texmf/web2c/pdftex",
"value": 3.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/mnt/var/lib/texmf/web2c/luatex",
"value": 3.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/mnt/var/cache/yum/aarch64/2/amzn2-core-source",
"value": 3.0,
"unit": "Megabytes",
"time": 1747067482108
}
]
},
"YarnNodeManager-CPU": {
"name": "YarnNodeManager-CPU",
"value": 0.5,
"unit": "Percent",
"time": 1747067545901
},
"Instance-Disk-Used-/dev/shm": {
"name": "Instance-Disk-Used-/dev/shm",
"value": 0.0,
"unit": "Percent",
"time": 1747067482108
},
"HadoopDataNode-RunTime": {
"name": "HadoopDataNode-RunTime",
"value": 2402.0,
"unit": "Second",
"time": 1747067545900
},
"InstanceController-RunTime": {
"name": "InstanceController-RunTime",
"value": 2470.0,
"unit": "Second",
"time": 1747067545901
},
"LogPusher-Total_ResendRate": {
"name": "LogPusher-Total_ResendRate",
"value": 1.0,
"unit": "Count",
"time": 1747067326139
},
"LogPusher-LastCycle-FailedFiles": {
"name": "LogPusher-LastCycle-FailedFiles",
"value": 0.0,
"unit": "Count",
"time": 1747067326139
},
"LogPusher-Reboots": {
"name": "LogPusher-Reboots",
"value": 0.0,
"unit": "Count",
"time": 1747065085915
},
"Customer-CPU": {
"name": "Customer-CPU",
"value": 0.5,
"unit": "Percent",
"time": 1747067545901
},
"Instance-Disk-Used-/boot/efi": {
"name": "Instance-Disk-Used-/boot/efi",
"value": 38.0,
"unit": "Percent",
"time": 1747067482108,
"subMetrics": [
{
"name": "/boot/efi",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
}
]
},
"Instance-Disk-Used-/emr": {
"name": "Instance-Disk-Used-/emr",
"value": 2.0,
"unit": "Percent",
"time": 1747067482108,
"subMetrics": [
{
"name": "/emr/setup-devices",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/emr/logpusher/run",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/emr/logpusher/log",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/emr/logpusher/db",
"value": 1.0,
 "unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/emr/instance-state",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/emr/instance-controller/run",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/emr/instance-controller/log/bootstrap-actions/1",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/emr/instance-controller/log",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/emr/instance-controller/lib/sslKeys",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/emr/instance-controller/lib/info",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
}
]
},
"YarnNodeManager-RunTime": {
"name": "YarnNodeManager-RunTime",
"value": 2399.0,
"unit": "Second",
"time": 1747067545901
},
"LogPusher-Memory": {
"name": "LogPusher-Memory",
"value": 1.6,
"unit": "Percent",
"time": 1747067545901
},
"InstanceController-CPU": {
"name": "InstanceController-CPU",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
 "Customer-CPU-Top10": {
"name": "Customer-CPU-Top10",
"value": 0.5,
"unit": "Percent",
"time": 1747067545901,
"subMetrics": [
{
"name": "top-CPU",
"value": 0.5,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "/usr/lib/systemd/systemd --switched-root --system --deserialize
21-CPU",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "[kthreadd]-CPU",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "[kworker/0:0H]-CPU",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "[kworker/u16:0]-CPU",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "[mm_percpu_wq]-CPU",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "[ksoftirqd/0]-CPU",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "[rcu_sched]-CPU",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
},
{
"name": "[rcu_bh]-CPU",
"value": 0.0,
"unit": "Percent",
 "time": 1747067545901
},
{
"name": "[migration/0]-CPU",
"value": 0.0,
"unit": "Percent",
"time": 1747067545901
}
]
},
"Instance-Disk-Used-/run": {
"name": "Instance-Disk-Used-/run",
"value": 1.0,
"unit": "Percent",
"time": 1747067482108,
"subMetrics": [
{
"name": "/run/udev/data",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/run/tmpfiles.d",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/run/systemd/users",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/run/systemd/system/user-0.slice.d",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/run/systemd/system/session-7.scope.d",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/run/systemd/system/session-3.scope.d",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/run/systemd/system",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
 "name": "/run/systemd/sessions",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/run/systemd/seats",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/run/systemd/journal/streams",
"value": 1.0,
"unit": "Megabytes",
"time": 1747067482108
}
]
},
"LogPusher-LastCycle-IgnoredFiles": {
"name": "LogPusher-LastCycle-IgnoredFiles",
"value": 0.0,
"unit": "Count",
"time": 1747067326139
},
"Instance-Memory-MemFree": {
"name": "Instance-Memory-MemFree",
"value": 11880.8,
"unit": "Megabytes",
"time": 1747067537724
},
"EMR-CPU": {
"name": "EMR-CPU",
"value": 1.0,
"unit": "Percent",
"time": 1747067545901
},
"Customer-Memory": {
"name": "Customer-Memory",
"value": 2.3000000000000003,
"unit": "Percent",
"time": 1747067545901
},
"HadoopDataNode-CPU": {
"name": "HadoopDataNode-CPU",
"value": 0.5,
"unit": "Percent",
"time": 1747067545900
},
"LogPusher-RunTime": {
"name": "LogPusher-RunTime",
"value": 2470.0,
"unit": "Second",
"time": 1747067545901
},
"LogPusher-LastCycle-PushDuration": {
"name": "LogPusher-LastCycle-PushDuration",
"value": 586.0,
"unit": "Milliseconds",
 "time": 1747067326139
},
"Instance-Disk-Used-/": {
"name": "Instance-Disk-Used-/",
"value": 73.0,
"unit": "Percent",
"time": 1747067482108,
"subMetrics": [
{
"name": "/usr/lib/hudi",
"value": 595.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/usr/bin",
"value": 493.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/usr/lib/hudi/cli/lib",
"value": 364.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/usr/share/aws/emr/hadoop-state-pusher/lib",
"value": 351.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/usr/share/aws/emr/emr-log-analytics-metrics/lib",
"value": 348.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/usr/share/aws/emr/emr-metrics-collector/lib",
"value": 335.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/usr/lib/spark/jars",
"value": 291.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/usr/share/aws/aws-java-sdk",
"value": 278.0,
"unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/usr/lib64",
"value": 217.0,
 "unit": "Megabytes",
"time": 1747067482108
},
{
"name": "/usr/lib/jvm/java-11-amazon-corretto.aarch64/lib",
"value": 206.0,
"unit": "Megabytes",
"time": 1747067482108
}
]
},
"HadoopDataNode-Memory": {
"name": "HadoopDataNode-Memory",
"value": 1.6,
"unit": "Percent",
"time": 1747067545900
},
"Instance-Disk-Used-/dev": {
"name": "Instance-Disk-Used-/dev",
"value": 0.0,
"unit": "Percent",
"time": 1747067482107
},
"HadoopDataNode-Reboots": {
"name": "HadoopDataNode-Reboots",
"value": 0.0,
"unit": "Count",
"time": 1747065145904
},
"InstanceController-Reboots": {
"name": "InstanceController-Reboots",
"value": 0.0,
"unit": "Count",
"time": 1747065085913
},
"LogPusher-Total-Logfile": {
"name": "LogPusher-Total-Logfile",
"value": 68.0,
"unit": "Count",
"time": 1747067326139
},
"LogPusher-LastCycle-UploadedFiles": {
"name": "LogPusher-LastCycle-UploadedFiles",
"value": 4.0,
"unit": "Count",
"time": 1747067326139
},
"IC-TimeSinceHeartbeat": {
"name": "IC-TimeSinceHeartbeat",
"value": 57.0,
"unit": "Seconds",
"time": 1747067545727
},
"LogPusher-LastCycle-DataSize": {
"name": "LogPusher-LastCycle-DataSize",
"value": 31415.0,
"unit": "Bytes",
"time": 1747067326139
},
 "Instance-CPU-CPUUtilization": {
"name": "Instance-CPU-CPUUtilization",
"value": 0.25,
"unit": "Percent",
"time": 1747067540729
}
}