QEMU Linux VM Locks Up!

[gflarity]

I believe I've found a bug with the Qemu/KVM stuff. I'm running
Ubuntu VMs and I can reliably get a VM to 'lock up' during boot.
Albeit the one that locks up is random.

Symptom: The CPU usage for the qemu process hits 12-13%, which is
equivalent to a core on my system. The system is unresponsive and
it'll stay that way indefinitely.

The funny thing is that the VM can be 'unlocked', either by running
gcore against the pid, or even by just running 'vmadm info' command
against the VM. It comes back instantly and finishes loading so after.

To reproduce the issue I just shut down a bunch of VMs then boot them
in succession. Usually one of say 6 will be in this 'locked' state.

I tried to use mdb -k to get more info, not sure if this is correct though:
[root@4c-72-b9-20-e1-5f ~]# mdb -k
Loading modules: [ unix genunix specfs dtrace mac cpu.generic uppc
pcplusmp scsi_vhci ufs ip hook neti sockfs arp usba stmf_sbd stmf zfs
sd lofs sata idm crypto random cpc logindmux ptm kvm sppp nsmb smbsrv
nfs ]

::pgrep qemu
S PID PPID PGID SID UID FLAGS ADDR NAME
R 18334 18284 18284 18284 0 0x4a004000 ffffff0911163010
qemu-system-x86_
R 18214 18164 18164 18164 0 0x4a004000 ffffff090f17a0a8
qemu-system-x86_
R 18094 18044 18044 18044 0 0x4a004000 ffffff09128e8018
qemu-system-x86_
R 17974 17924 17924 17924 0 0x4a004000 ffffff090f1390a0
qemu-system-x86_
R 17854 17804 17804 17804 0 0x4a004000 ffffff09117650c8
qemu-system-x86_
R 17734 17684 17684 17684 0 0x4a004000 ffffff091266e008
qemu-system-x86_
R 17614 17564 17564 17564 0 0x4a004000 ffffff0910168000
qemu-system-x86_
R 17494 17444 17444 17444 0 0x4a004000 ffffff09146500c0
qemu-system-x86_
R 17374 17324 17324 17324 0 0x4a004000 ffffff09086480c8
qemu-system-x86_
R 17254 17204 17204 17204 0 0x4a004000 ffffff090f4d7000
qemu-system-x86_
R 4767 3968 3968 3968 0 0x4a004000 ffffff090fd54080
qemu-system-x86_
R 4680 3941 3941 3941 0 0x4a004000 ffffff09161db030
qemu-system-x86_
R 4500 3686 3686 3686 0 0x4a004000 ffffff0913efc068
qemu-system-x86_
R 4366 3599 3599 3599 0 0x4a004000 ffffff0910d250a8
qemu-system-x86_
R 4745 3498 3498 3498 0 0x4a004000 ffffff0911fce0d0
qemu-system-x86_
R 3743 3365 3365 3365 0 0x4a004000 ffffff091176d0c0
qemu-system-x86_
R 4094 3337 3337 3337 0 0x4a004000 ffffff0913d7d058
qemu-system-x86_
R 4333 3133 3133 3133 0 0x4a004000 ffffff09147a20d0
qemu-system-x86_
R 3550 3121 3121 3121 0 0x4a004000 ffffff090fd09060
qemu-system-x86_
ffffff090f1390a0::threadlist
ADDR PROC LWP CMD/LWPID
ffffff090f1390a0 ffffff09d9447e90 ffffff0909f8f3e0 0xffffff09013e7840()
ffffff09d9447e90::findstack
mdb: thread ffffff09d9447e90 isn't in memory
ffffff0909f8f3e0::findstack
stack pointer for thread ffffff0909f8f3e0: ffffff003fb62c60
[ ffffff003fb62c60 _resume_from_idle+0xf1() ]
ffffff003fb62c90 swtch+0x141()
ffffff003fb62d30 cv_timedwait_sig_hires+0x39d()
ffffff003fb62d60 cv_reltimedwait_sig+0x4f()
ffffff003fb62d90 cv_relwaituntil_sig+0x25()
ffffff003fb62e60 poll_common+0x569()
ffffff003fb62f00 pollsys+0xe3()
ffffff003fb62f10 sys_syscall+0x17a()

[rmustacc]

Can you please share the version of the platform you are running on and the output of vmadm get for the VMs?

[gflarity]

Absolutely, if ssh access would help at all, I can provide that too:

Last login: Tue Sep 11 18:38:59 2012 from 172.24.3.146

• SmartOS Live Image v0.147+ build: 20120906T221231Z

{
"zonename": "df018086-e0d5-4f10-ab71-9a7b0594528d",
"zonepath": "/zones/df018086-e0d5-4f10-ab71-9a7b0594528d",
"autoboot": true,
"brand": "kvm",
"limit_priv": "default,-file_link_any,-net_access,-proc_fork,-proc_info,-proc_session",
"create_timestamp": "2012-08-29T16:42:02.604Z",
"cpu_shares": 100,
"zfs_io_priority": 100,
"max_lwps": 2000,
"max_physical_memory": 2048,
"max_locked_memory": 2048,
"max_swap": 2048,
"billing_id": "00000000-0000-0000-0000-000000000000",
"owner_uuid": "00000000-0000-0000-0000-000000000000",
"hostname": "manage",
"resolvers": [
"8.8.8.8"
],
"alias": "manage.well.tor",
"ram": 1024,
"vcpus": 4,
"nics": [
{
"interface": "net0",
"mac": "c2:6a:3d:01:0b:13",
"nic_tag": "admin",
"gateway": "172.24.3.1",
"primary": true,
"ip": "172.24.3.81",
"netmask": "255.255.0.0",
"model": "virtio"
}
],
"disks": [
{
"path": "/dev/zvol/rdsk/zones/df018086-e0d5-4f10-ab71-9a7b0594528d-disk0",
"boot": true,
"model": "virtio",
"media": "disk",
"size": 16500,
"image_uuid": "00000000-0000-0000-0000-000000000000",
"zfs_filesystem": "zones/df018086-e0d5-4f10-ab71-9a7b0594528d-disk0",
"zpool": "zones",
"compression": "off",
"block_size": 8192
}
],
"uuid": "df018086-e0d5-4f10-ab71-9a7b0594528d",
"last_modified": "2012-09-11T18:58:13.000Z",
"server_uuid": "df25bb06-b03c-11e1-890b-505054503030",
"zoneid": 84,
"pid": 17620,
"zone_state": "running",
"state": "running",
"quota": 10,
"zfs_root_recsize": 131072,
"zfs_filesystem": "zones/df018086-e0d5-4f10-ab71-9a7b0594528d",
"zpool": "zones",
"customer_metadata": {},
"internal_metadata": {},
"tags": {}
}